From 68bcf76033e35a8a0e0fd4017575a9fde14398d7 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Sat, 5 Sep 2026 13:49:56 -0500 Subject: [PATCH 1/5] 20260904-XMEMALIGN WIP --- .wolfssl_known_macro_extras | 2 + linuxkm/linuxkm_wc_port.h | 7 + src/pk.c | 21 +- wolfcrypt/src/integer.c | 8 +- wolfcrypt/src/memory.c | 177 ++++++++++++++ wolfcrypt/test/test.c | 325 ++++++++++++++++++++++++-- wolfssl/wolfcrypt/memory.h | 13 ++ wolfssl/wolfcrypt/types.h | 455 ++++++++++++++++++++++++++++-------- 8 files changed, 874 insertions(+), 134 deletions(-) diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 02a88c29308..83af5346fe0 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -1110,6 +1110,7 @@ WOLFSSL_NO_KTRI_ORACLE_WARNING WOLFSSL_NO_LMS_SHAKE256_256 WOLFSSL_NO_OCSP_DATE_CHECK WOLFSSL_NO_OCSP_OPTIONAL_CERTS +WOLFSSL_NO_POSIX_MEMALIGN WOLFSSL_NO_RSA_KEY_CHECK WOLFSSL_NO_RSA_SW WOLFSSL_NO_SERVER_GROUPS_EXT @@ -1348,6 +1349,7 @@ _UINTPTR_T_DECLARED _WIN32 _WIN32_WCE _WIN64 +_XOPEN_SOURCE _XOPEN_SOURCE_EXTENDED __32MZ2048ECH144__ __32MZ2048ECM144__ diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 8b2abcc2252..16dd555da4d 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -2419,6 +2419,13 @@ #if defined(USE_KVREALLOC) || !defined(USE_KVMALLOC) #define XREALLOC(p, n, h, t) __extension__ ({(void)(h); (void)(t); realloc(p, n);}) #endif + /* The default linuxkm malloc wrappers (see above) assure adequate + * alignment. In particular we do not need to worry about alignments + * larger than the allocation size, so rounded-up-size-based alignment + * is enough. */ + #define XMEMALIGN(a, s, h, t) ((void)(a), (void)(h), (void)(t), malloc(s)) + #define XFREEALIGN(p, h, t) do { (void)(h); (void)(t); free(p); } while (0) + #endif /* !XMALLOC_USER && !XMALLOC_OVERRIDE */ #endif diff --git a/src/pk.c b/src/pk.c index 8b91ff04268..975c7d59126 100644 --- a/src/pk.c +++ b/src/pk.c @@ -495,20 +495,13 @@ static int der_to_enc_pem_alloc(unsigned char* der, int derSz, int blockSz = wolfSSL_EVP_CIPHER_block_size(cipher); byte *tmpBuf; - /* Add space for padding. */ - #ifdef WOLFSSL_NO_REALLOC - tmpBuf = (byte*)XMALLOC((size_t)(derSz + blockSz), heap, - DYNAMIC_TYPE_TMP_BUFFER); - if (tmpBuf != NULL && der != NULL) - { - XMEMCPY(tmpBuf, der, (size_t)(derSz)); - XFREE(der, heap, DYNAMIC_TYPE_TMP_BUFFER); - der = NULL; - } - #else - tmpBuf = (byte*)XREALLOC(der, (size_t)(derSz + blockSz), heap, - DYNAMIC_TYPE_TMP_BUFFER); - #endif + /* Add space for padding. der holds a plaintext private key DER: + * XREALLOC_SCRUBBED() ForceZero()s the old allocation before + * releasing it, so no key bytes are abandoned on the free list. + * (It never uses native realloc(), so it also subsumes the + * WOLFSSL_NO_REALLOC case.) */ + tmpBuf = (byte*)XREALLOC_SCRUBBED(der, (size_t)derSz, + (size_t)(derSz + blockSz), heap, DYNAMIC_TYPE_TMP_BUFFER); if (tmpBuf == NULL) { WOLFSSL_ERROR_MSG("Extending DER buffer failed"); ret = 0; /* der buffer is free'd at the end of the function */ diff --git a/wolfcrypt/src/integer.c b/wolfcrypt/src/integer.c index ccf7e2c2fe8..398d788a026 100644 --- a/wolfcrypt/src/integer.c +++ b/wolfcrypt/src/integer.c @@ -399,7 +399,13 @@ int mp_grow (mp_int * a, int size) * in case the operation failed we don't want * to overwrite the dp member of a. */ - tmp = (mp_digit *)XREALLOC (a->dp, sizeof (mp_digit) * size, NULL, + /* mp_ints routinely hold private keys and other secret values, so the + * old digit array must not be abandoned on the free list with live + * limbs: XREALLOC_SCRUBBED() ForceZero()s it before releasing it. This + * also scrubs public values, an acceptable cost for correctness. */ + tmp = (mp_digit *)XREALLOC_SCRUBBED (a->dp, + sizeof (mp_digit) * a->alloc, + sizeof (mp_digit) * size, NULL, DYNAMIC_TYPE_BIGINT); if (tmp == NULL) { /* reallocation failed but "a" is still valid [can be freed] */ diff --git a/wolfcrypt/src/memory.c b/wolfcrypt/src/memory.c index a2df2258ebe..27ca3a34fa5 100644 --- a/wolfcrypt/src/memory.c +++ b/wolfcrypt/src/memory.c @@ -486,6 +486,69 @@ void wolfSSL_Free(void *ptr) } } +#ifndef WC_XMEMALIGN_PORT_OVERRIDE +/* Aligned allocation wrappers over the wc_xmemalign()/wc_xfreealign() + * bottommost primitives. When the primitives' over-allocation branch is in + * effect, the underlying memory comes from XMALLOC/XFREE, i.e. + * wolfSSL_Malloc()/wolfSSL_Free() here, so allocator callbacks and all + * allocation accounting apply automatically. When the native + * posix_memalign() branch is in effect (WC_XMEMALIGN_NATIVE), the allocation + * bypasses wolfSSL_Malloc(): WOLFSSL_MEM_FAIL_COUNT accounting is applied + * here instead, while registered allocator callbacks and + * WOLFSSL_CHECK_MEM_ZERO coverage deliberately do not extend to aligned + * allocations in that configuration. */ +#ifdef WOLFSSL_DEBUG_MEMORY +void* wolfSSL_MemAlign(size_t alignment, size_t size, const char* func, + unsigned int line) +#else +void* wolfSSL_MemAlign(size_t alignment, size_t size) +#endif +{ + void* res; + +#if defined(WC_XMEMALIGN_NATIVE) && defined(WOLFSSL_MEM_FAIL_COUNT) + if (!wc_MemFailCount_AllocMem()) { + WOLFSSL_MSG("MemFailCnt: Fail memalign"); + return NULL; + } +#endif + res = wc_xmemalign(alignment, size, NULL, DYNAMIC_TYPE_TMP_BUFFER); + +#ifdef WOLFSSL_DEBUG_MEMORY +#if defined(WOLFSSL_DEBUG_MEMORY_PRINT) && !defined(WOLFSSL_TRACK_MEMORY) + fprintf(stderr, "MemAlign: %p -> %u (align %u) at %s:%u\n", res, + (word32)size, (word32)alignment, func, line); +#else + (void)func; + (void)line; +#endif +#endif + + return res; +} + +#ifdef WOLFSSL_DEBUG_MEMORY +void wolfSSL_FreeAlign(void *ptr, const char* func, unsigned int line) +#else +void wolfSSL_FreeAlign(void *ptr) +#endif +{ +#ifdef WOLFSSL_DEBUG_MEMORY +#if defined(WOLFSSL_DEBUG_MEMORY_PRINT) && !defined(WOLFSSL_TRACK_MEMORY) + fprintf(stderr, "FreeAlign: %p at %s:%u\n", ptr, func, line); +#else + (void)func; + (void)line; +#endif +#endif +#if defined(WC_XMEMALIGN_NATIVE) && defined(WOLFSSL_MEM_FAIL_COUNT) + if (ptr != NULL) + wc_MemFailCount_FreeMem(); +#endif + wc_xfreealign(ptr, NULL, DYNAMIC_TYPE_TMP_BUFFER); +} +#endif /* !WC_XMEMALIGN_PORT_OVERRIDE */ + #ifdef WOLFSSL_DEBUG_MEMORY void* wolfSSL_Realloc(void *ptr, size_t size, const char* func, unsigned int line) #else @@ -1737,6 +1800,120 @@ void wc_ForceZero(void *mem, size_t len) } #endif +/* Reallocation with scrubbing and/or alignment guarantees: implementations + * behind the XREALLOC_SCRUBBED()/XREALLOCALIGN()/XREALLOCALIGN_SCRUBBED() + * macros (types.h), composed over the XREALLOC/XMALLOC/XFREE/XMEMALIGN/ + * XFREEALIGN macro bindings so that port overrides, wolfSSL memory + * callbacks, and static pools are all honored. See the contract notes at + * the macro definitions. On failure (NULL return) the original allocation + * is always intact and, for scrub, unscrubbed. */ +void *wc_xrealloc_scrubbed(void *ptr, size_t old_size, size_t size, + void *heap, int tag) +{ + void *new_p; + + (void)heap; + (void)tag; + + if (ptr == NULL) + return XMALLOC(size, heap, tag); + if (size == 0) { + /* realloc-to-zero idiom: scrub, free, and return NULL. Handled + * explicitly (and ahead of all other arms) so the semantics are + * uniform across builds rather than the underlying allocator's + * implementation-defined choice. */ +#ifdef WOLFSSL_NO_FORCE_ZERO + XMEMSET(ptr, 0, old_size); +#else + ForceZero(ptr, old_size); +#endif + XFREE(ptr, heap, tag); + return NULL; + } + new_p = XMALLOC(size, heap, tag); + if (new_p == NULL) + return NULL; /* original intact and unscrubbed */ + XMEMCPY(new_p, ptr, (old_size < size) ? old_size : size); +#ifdef WOLFSSL_NO_FORCE_ZERO + XMEMSET(ptr, 0, old_size); +#else + ForceZero(ptr, old_size); +#endif + XFREE(ptr, heap, tag); + return new_p; +} + +void *wc_xreallocalign(void *ptr, size_t alignment, size_t old_size, + size_t size, void *heap, int tag, int scrub) +{ + void *new_p; + + (void)heap; + if (size == 0) { + /* realloc-to-zero idiom: free (scrubbed if requested) and return + * NULL. Handled explicitly, ahead of the delegation and retention + * arms: delegation would inherit the underlying allocator's + * implementation-defined choice, and retention would wrongly keep a + * small allocation alive. */ + if (scrub) { +#ifdef WOLFSSL_NO_FORCE_ZERO + XMEMSET(ptr, 0, old_size); +#else + ForceZero(ptr, old_size); +#endif + } + XFREEALIGN(ptr, heap, tag); + return NULL; + } + + (void)tag; + + if (ptr == NULL) + return XMEMALIGN(alignment, size, heap, tag); + + /* Match the alignment policy of the allocation-side primitives, so the + * shrink-retention and delegation tests below reason about the alignment + * actually in effect. */ + if (alignment < sizeof(void *)) + alignment = sizeof(void *); + else if ((alignment & (alignment - 1U)) != 0) + return NULL; /* original intact */ + +#if defined(WC_XMEMALIGN_NATIVE) + /* posix_memalign() memory is first-class native-heap memory, and native + * realloc() results are guaranteed at least sizeof(void *) alignment, so + * requests at or below that can delegate wholesale. This arm is + * unavailable when XMEMALIGN() is the over-allocation shim (the pointer + * is interior to the underlying allocation) or a port override (unknown + * provenance). */ + if ((!scrub) && (alignment == sizeof(void *))) + return XREALLOC(ptr, size, heap, tag); +#endif + + /* Shrink retention: when the reclaimable slack cannot exceed the aligned + * allocator's own overhead, compaction is footprint-neutral at best -- + * retain the allocation. Larger shrinks fall through and genuinely + * release memory. (Nothing is released here, so no scrub is owed.) */ + if ((size <= old_size) && + ((old_size - size) <= ((alignment - 1U) + sizeof(word32))) && + (((wc_ptr_t)ptr & ((wc_ptr_t)alignment - 1U)) == 0)) + return ptr; + + new_p = XMEMALIGN(alignment, size, heap, tag); + if (new_p == NULL) + return NULL; /* original intact and unscrubbed */ + XMEMCPY(new_p, ptr, (old_size < size) ? old_size : size); + if (scrub) { +#ifdef WOLFSSL_NO_FORCE_ZERO + XMEMSET(ptr, 0, old_size); +#else + ForceZero(ptr, old_size); +#endif + } + XFREEALIGN(ptr, heap, tag); + return new_p; +} + #ifndef WOLFSSL_NO_CONST_CMP /* Exported version of ConstantCompare(). */ int wc_ConstantCompare(const byte* a, const byte* b, int length) diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 2ad6034e8a0..e733dbe009c 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -415,7 +415,7 @@ static const byte const_byte_array[] = "A+Gd\0\0\0"; #ifdef HAVE_ASCON #include #endif -#ifdef HAVE_ARGON2 +#if defined(HAVE_ARGON2) && !defined(WOLFSSL_NO_MALLOC) #include #endif #include @@ -26592,63 +26592,53 @@ typedef struct keywrapVector { #if !defined(HAVE_FIPS) && !defined(HAVE_SELFTEST) && \ !defined(WOLFSSL_NO_MALLOC) -/* struct Aes cannot be a local here: with --enable-aesgcm=table its GCM tables - * alone are 4096 bytes, past the frame limit CI enforces. It also asks for 16 - * byte alignment through its ALIGN16 members, which XMALLOC does not guarantee - * on 32-bit targets, where malloc only promises 8 -- and wc_AesSetIV() lowers - * to an alignment-qualified NEON store when building for armv8-a+crypto, which - * faults on an under-aligned object. Neither a plain local nor a plain XMALLOC - * (wc_AesNew() included) satisfies both, so allocate with headroom, align by - * hand, and keep the base pointer for XFREE. */ #define AESKEYWRAP_AES_ALIGN 16 static wc_test_ret_t aeskeywrap_caller_aes_test(const keywrapVector* v, byte* output, word32 outputSz, byte* plain, word32 plainBufSz) { - void* raw; - Aes* aes; + WC_DECLARE_VAR_ALIGNED(aes, Aes, AESKEYWRAP_AES_ALIGN, 1, HEAP_HINT); wc_test_ret_t ret = 0; int wrapSz = 0, plainSz; - raw = XMALLOC(sizeof(Aes) + AESKEYWRAP_AES_ALIGN - 1, HEAP_HINT, - DYNAMIC_TYPE_TMP_BUFFER); - if (raw == NULL) - return WC_TEST_RET_ENC_NC; - aes = (Aes*)(void*)(((wc_ptr_t)raw + (AESKEYWRAP_AES_ALIGN - 1)) & - ~(wc_ptr_t)(AESKEYWRAP_AES_ALIGN - 1)); + /* Alignment here is semantic, not advisory: armv8-a+crypto uses + * alignment-qualified NEON stores that fault on an under-aligned Aes. + * XMALLOC guarantees only 8. */ + WC_ALLOC_VAR_ALIGNED_EX(aes, Aes, AESKEYWRAP_AES_ALIGN, 1, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER, return WC_TEST_RET_ENC_EC(MEMORY_E)); XMEMSET(output, 0, outputSz); XMEMSET(plain, 0, plainBufSz); if (wc_AesInit(aes, HEAP_HINT, devId) != 0) { - XFREE(raw, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + WC_FREE_VAR_ALIGNED_EX(aes, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); return WC_TEST_RET_ENC_NC; } - if (wc_AesSetKey(aes, v->kek, v->kekLen, NULL, AES_ENCRYPTION) != 0) - ret = WC_TEST_RET_ENC_NC; + if ((ret = wc_AesSetKey(aes, v->kek, v->kekLen, NULL, AES_ENCRYPTION)) != 0) + ret = WC_TEST_RET_ENC_EC(ret); if (ret == 0) { wrapSz = wc_AesKeyWrap_ex(aes, v->data, v->dataLen, output, outputSz, NULL); if ((wrapSz < 0) || (wrapSz != (int)v->verifyLen) || XMEMCMP(output, v->verify, v->verifyLen) != 0) { - ret = WC_TEST_RET_ENC_NC; + ret = WC_TEST_RET_ENC_EC(wrapSz); } } if (ret == 0) { - if (wc_AesSetKey(aes, v->kek, v->kekLen, NULL, AES_DECRYPTION) != 0) - ret = WC_TEST_RET_ENC_NC; + if ((ret = wc_AesSetKey(aes, v->kek, v->kekLen, NULL, AES_DECRYPTION)) != 0) + ret = WC_TEST_RET_ENC_EC(ret); } if (ret == 0) { plainSz = wc_AesKeyUnWrap_ex(aes, output, (word32)wrapSz, plain, plainBufSz, NULL); if ((plainSz < 0) || (plainSz != (int)v->dataLen) || XMEMCMP(plain, v->data, v->dataLen) != 0) { - ret = WC_TEST_RET_ENC_NC; + ret = WC_TEST_RET_ENC_EC(plainSz); } } + wc_AesFree(aes); - XFREE(raw, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + WC_FREE_VAR_ALIGNED_EX(aes, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); return ret; } #endif @@ -37875,6 +37865,230 @@ static int simple_mem_test(size_t sz) XFREE(b, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); return ret; } + +static wc_test_ret_t simple_memalign_test(size_t alignment, size_t sz) +{ + wc_test_ret_t ret = 0; + byte* b; + int i; + + b = (byte*)XMEMALIGN(alignment, sz, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (b == NULL) { + return WC_TEST_RET_ENC_I((word32)alignment); + } + /* The requested alignment is the contract; the generic implementation's + * flooring of small values is not asserted, so only power-of-2 requests + * >= 2 are mask-checked. */ + if ((alignment >= 2) && ((alignment & (alignment - 1U)) == 0) && + (((wc_ptr_t)b & ((wc_ptr_t)alignment - 1U)) != 0)) { + ret = WC_TEST_RET_ENC_I((word32)alignment); + } + if (ret == 0) { + /* utilize the full allocation: catches offset-slot overlap in the + * over-allocation implementation. */ + for (i = 0; i < (int)sz; i++) { + b[i] = (byte)i; + } + for (i = 0; i < (int)sz; i++) { + if (b[i] != (byte)i) { + ret = WC_TEST_RET_ENC_I((word32)i); + break; + } + } + } + XFREEALIGN(b, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return ret; +} + +#define WC_TEST_MA_ALIGN 64U +#define WC_TEST_MA_SZ 256U /* multiple of WC_TEST_MA_ALIGN: see the stride + * asserts in WC_DECLARE_ARRAY_ALIGNED. */ +static wc_test_ret_t memalign_macros_test(void) +{ + wc_test_ret_t ret = 0; + int i; + WC_DECLARE_VAR_ALIGNED(vbuf, byte, WC_TEST_MA_ALIGN, WC_TEST_MA_SZ, + HEAP_HINT); + WC_DECLARE_ARRAY_ALIGNED(abuf, byte, 3, WC_TEST_MA_ALIGN, WC_TEST_MA_SZ, + HEAP_HINT); + + WC_ALLOC_VAR_ALIGNED_EX(vbuf, byte, WC_TEST_MA_ALIGN, WC_TEST_MA_SZ, + HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER, + return WC_TEST_RET_ENC_EC(MEMORY_E)); + WC_ALLOC_ARRAY_ALIGNED(abuf, byte, 3, WC_TEST_MA_ALIGN, WC_TEST_MA_SZ, + HEAP_HINT); + if (!WC_ARRAY_OK(abuf)) { + ret = WC_TEST_RET_ENC_EC(MEMORY_E); + } + + /* XALIGNED() expands to nothing on compilers with no known alignment + * attribute, so only assert alignment where the declaration (stack + * build) or XMEMALIGN() (heap build) actually guarantees it. */ +#if defined(WC_DECLARE_VAR_IS_HEAP_ALLOC) || defined(__GNUC__) || \ + defined(__CC_ARM) || defined(_MSC_VER) + if ((ret == 0) && (((wc_ptr_t)vbuf & (WC_TEST_MA_ALIGN - 1U)) != 0)) { + ret = WC_TEST_RET_ENC_NC; + } + if (ret == 0) { + for (i = 0; i < 3; i++) { + if (((wc_ptr_t)abuf[i] & (WC_TEST_MA_ALIGN - 1U)) != 0) { + ret = WC_TEST_RET_ENC_I((word32)i); + break; + } + } + } +#endif + + if (ret == 0) { + /* utilize every byte, then spot-check for cross-item clobbering. */ + XMEMSET(vbuf, 0x5A, WC_TEST_MA_SZ); + for (i = 0; i < 3; i++) { + XMEMSET(abuf[i], 0xA0 + i, WC_TEST_MA_SZ); + } + if ((vbuf[0] != 0x5A) || (vbuf[WC_TEST_MA_SZ - 1U] != 0x5A)) { + ret = WC_TEST_RET_ENC_NC; + } + for (i = 0; (ret == 0) && (i < 3); i++) { + if ((abuf[i][0] != (byte)(0xA0 + i)) || + (abuf[i][WC_TEST_MA_SZ - 1U] != (byte)(0xA0 + i))) { + ret = WC_TEST_RET_ENC_I((word32)i); + } + } + } + + if (ret == 0) { + /* CALLOC variant: reallocate vbuf zeroed. */ + WC_FREE_VAR_ALIGNED_EX(vbuf, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + WC_CALLOC_VAR_ALIGNED_EX(vbuf, byte, WC_TEST_MA_ALIGN, WC_TEST_MA_SZ, + HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER, + ret = WC_TEST_RET_ENC_EC(MEMORY_E)); + } + if (ret == 0) { +#ifdef WC_DECLARE_VAR_IS_HEAP_ALLOC + /* heap arm zeroes the full extent. */ + for (i = 0; i < (int)WC_TEST_MA_SZ; i++) { + if (vbuf[i] != 0) { + ret = WC_TEST_RET_ENC_I((word32)i); + break; + } + } +#else + /* stack arm's CALLOC zeroes only sizeof(VAR_TYPE) (pre-existing + * behavior mirrored from WC_CALLOC_VAR): only [0] is guaranteed. */ + if (vbuf[0] != 0) { + ret = WC_TEST_RET_ENC_NC; + } +#endif + } + + WC_FREE_VAR_ALIGNED_EX(vbuf, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + WC_FREE_ARRAY_ALIGNED(abuf, 3, HEAP_HINT); + return ret; +} + +static wc_test_ret_t memalign_realloc_test(void) +{ + wc_test_ret_t ret = 0; + byte* p; + byte* q; + int i; + + /* NULL in: degrades to fresh aligned allocation. */ + p = (byte*)XREALLOCALIGN(NULL, 64U, 0U, 300U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + if (((wc_ptr_t)p & 63U) != 0) { + XFREEALIGN(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_NC; + } + for (i = 0; i < 300; i++) + p[i] = (byte)i; + + /* scrubbed grow: contents and alignment preserved. */ + q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 300U, 1600U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q == NULL) { + /* original intact on failure, per contract. */ + XFREEALIGN(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_EC(MEMORY_E); + } + if (((wc_ptr_t)q & 63U) != 0) + ret = WC_TEST_RET_ENC_NC; + for (i = 0; (ret == 0) && (i < 300); i++) { + if (q[i] != (byte)i) + ret = WC_TEST_RET_ENC_I((word32)i); + } + + /* small shrink: retention (pointer identity), reclaimable slack below + * the aligned allocator's own overhead. */ + if (ret == 0) { + p = (byte*)XREALLOCALIGN(q, 64U, 1600U, 1590U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (p != q) + ret = WC_TEST_RET_ENC_NC; + else + q = p; + } + + /* large shrink: compaction (may move); contents and alignment + * preserved. */ + if (ret == 0) { + p = (byte*)XREALLOCALIGN(q, 64U, 1600U, 128U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) { + ret = WC_TEST_RET_ENC_EC(MEMORY_E); + p = q; /* original intact on failure */ + } + else { + q = p; + if (((wc_ptr_t)q & 63U) != 0) + ret = WC_TEST_RET_ENC_NC; + for (i = 0; (ret == 0) && (i < 128); i++) { + if (q[i] != (byte)i) + ret = WC_TEST_RET_ENC_I((word32)i); + } + } + } + XFREEALIGN(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (ret != 0) + return ret; + + /* unaligned scrubbed roundtrip. */ + p = (byte*)XMALLOC(200U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + for (i = 0; i < 200; i++) + p[i] = (byte)(255 - i); + q = (byte*)XREALLOC_SCRUBBED(p, 200U, 400U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q == NULL) { + XFREE(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_EC(MEMORY_E); + } + for (i = 0; (ret == 0) && (i < 200); i++) { + if (q[i] != (byte)(255 - i)) + ret = WC_TEST_RET_ENC_I((word32)i); + } + XFREE(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + + /* realloc-to-zero idiom: frees and returns NULL. */ + if (ret == 0) { + p = (byte*)XMEMALIGN(64U, 128U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + if (XREALLOCALIGN_SCRUBBED(p, 64U, 128U, 0U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER) != NULL) + return WC_TEST_RET_ENC_NC; + p = (byte*)XMALLOC(96U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + if (XREALLOC_SCRUBBED(p, 96U, 0U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER) != NULL) + return WC_TEST_RET_ENC_NC; + } + return ret; +} #endif /* If successful, returns the first letter of the byte array `in`. @@ -38041,6 +38255,67 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memory_test(void) } #endif +#if defined(WOLFSSL_STATIC_MEMORY) || !defined(WOLFSSL_NO_MALLOC) + /* aligned allocation roundtrips. Every allocation is kept at least as + * large as its alignment: port overrides (e.g. linuxkm) promise only + * size-based natural alignment. */ + { + static const size_t memalign_aligns[] = { + 0U, 1U, 2U, 3U, sizeof(void *), 16U, 64U, 256U + }; + word32 ma_i; + for (ma_i = 0; + ma_i < (word32)(sizeof(memalign_aligns)/sizeof(memalign_aligns[0])); + ma_i++) { + size_t sz = (size_t)MEM_TEST_SZ; + if (sz < memalign_aligns[ma_i]) + sz = memalign_aligns[ma_i]; + ret = simple_memalign_test(memalign_aligns[ma_i], sz); + if (ret != 0) + return ret; + } + } + + /* WC_*_VAR_ALIGNED / WC_*_ARRAY_ALIGNED macro family */ + ret = memalign_macros_test(); + if (ret != 0) + return ret; + + /* XREALLOC_SCRUBBED / XREALLOCALIGN / XREALLOCALIGN_SCRUBBED */ + ret = memalign_realloc_test(); + if (ret != 0) + return ret; + +#ifndef WC_XMEMALIGN_PORT_OVERRIDE + /* Bogus-alignment rejection and overflow resilience are generic- + * implementation semantics, not part of the port-override contract. */ + if (XMEMALIGN(24U, (size_t)MEM_TEST_SZ, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER) != NULL) { + return WC_TEST_RET_ENC_NC; /* non-power-of-2 alignment accepted */ + } +#ifndef WC_XMEMALIGN_NATIVE + /* The over-allocation implementation must reject a size whose padded + * total wraps, without ever reaching the underlying allocator (keeps + * sanitizer runs quiet). */ + if (XMEMALIGN(64U, (size_t)0 - 32U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER) != NULL) { + return WC_TEST_RET_ENC_NC; /* padded-size overflow accepted */ + } +#endif +#endif /* !WC_XMEMALIGN_PORT_OVERRIDE */ +#endif /* WOLFSSL_STATIC_MEMORY || !WOLFSSL_NO_MALLOC */ + +#if defined(WOLFSSL_NO_MALLOC) && !defined(WOLFSSL_STATIC_MEMORY) && \ + !defined(WC_XMEMALIGN_PORT_OVERRIDE) + /* stub configuration: aligned allocation must fail cleanly */ + if (XMEMALIGN(16U, 16U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER) != NULL) { + return WC_TEST_RET_ENC_NC; + } +#endif + + /* NULL tolerance, all configurations */ + XFREEALIGN(NULL, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + #if !defined(USE_FAST_MATH) && !defined(WOLFSSL_NO_MALLOC) && defined(XREALLOC) /* realloc test */ { diff --git a/wolfssl/wolfcrypt/memory.h b/wolfssl/wolfcrypt/memory.h index 91200566acb..f1ec8cc0333 100644 --- a/wolfssl/wolfcrypt/memory.h +++ b/wolfssl/wolfcrypt/memory.h @@ -77,6 +77,8 @@ WOLFSSL_API void* wolfSSL_Malloc(size_t size, const char* func, unsigned int line); WOLFSSL_API void wolfSSL_Free(void *ptr, const char* func, unsigned int line); WOLFSSL_API void* wolfSSL_Realloc(void *ptr, size_t size, const char* func, unsigned int line); + WOLFSSL_API void* wolfSSL_MemAlign(size_t alignment, size_t size, const char* func, unsigned int line); + WOLFSSL_API void wolfSSL_FreeAlign(void *ptr, const char* func, unsigned int line); #else typedef void *(*wolfSSL_Malloc_cb)(size_t size); typedef void (*wolfSSL_Free_cb)(void *ptr); @@ -85,6 +87,8 @@ WOLFSSL_API void* wolfSSL_Malloc(size_t size); WOLFSSL_API void wolfSSL_Free(void *ptr); WOLFSSL_API void* wolfSSL_Realloc(void *ptr, size_t size); + WOLFSSL_API void* wolfSSL_MemAlign(size_t alignment, size_t size); + WOLFSSL_API void wolfSSL_FreeAlign(void *ptr); #endif /* WOLFSSL_DEBUG_MEMORY */ #endif /* WOLFSSL_STATIC_MEMORY */ @@ -333,6 +337,15 @@ WOLFSSL_LOCAL void wc_MemZero_Check(void* addr, size_t len); #ifndef WOLFSSL_NO_FORCE_ZERO WOLFSSL_API void wc_ForceZero(void *mem, size_t len); + +/* implementations behind XREALLOC_SCRUBBED()/XREALLOCALIGN()/ + * XREALLOCALIGN_SCRUBBED() (types.h) -- see contract notes there. + * Guarded with wc_ForceZero(): the scrub arms require ForceZero(). */ +WOLFSSL_API void *wc_xrealloc_scrubbed(void *ptr, size_t old_size, + size_t size, void *heap, int tag); +WOLFSSL_API void *wc_xreallocalign(void *ptr, size_t alignment, + size_t old_size, size_t size, void *heap, + int tag, int scrub); #endif #ifndef WOLFSSL_NO_CONST_CMP diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index b4b67cd1f33..566fc7a0330 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -756,6 +756,15 @@ enum { ((out) = (in1) - (in2), \ /* coverity[INTEGER_OVERFLOW] */ 1)))) +#if defined(XMEMALIGN) != defined(XFREEALIGN) + #error XMEMALIGN and XFREEALIGN must be defined as a matched pair. +#endif +#ifdef XMEMALIGN + /* A port (e.g. linuxkm) supplied the aligned-allocation wrappers before + * this platform chain; suppress the generic implementations below. */ + #define WC_XMEMALIGN_PORT_OVERRIDE +#endif + #if defined(HAVE_IO_POOL) WOLFSSL_API void* XMALLOC(size_t n, void* heap, int type); WOLFSSL_API void* XREALLOC(void *p, size_t n, void* heap, int type); @@ -929,6 +938,12 @@ enum { #endif #define XREALLOC(p, n, h, t) ((void)(h), (void)(t), \ wolfSSL_Realloc((p), (n), __func__, __LINE__)) + #ifndef WC_XMEMALIGN_PORT_OVERRIDE + #define XMEMALIGN(a, s, h, t) ((void)(h), (void)(t), \ + wolfSSL_MemAlign((a), (s), __func__, __LINE__)) + #define XFREEALIGN(p, h, t) do { (void)(h); (void)(t); \ + wolfSSL_FreeAlign((p), __func__, __LINE__); } while (0) + #endif #else #define XMALLOC(s, h, t) ((void)(h), (void)(t), \ wolfSSL_Malloc((s))) @@ -941,6 +956,12 @@ enum { #endif #define XREALLOC(p, n, h, t) ((void)(h), (void)(t), \ wolfSSL_Realloc((p), (n))) + #ifndef WC_XMEMALIGN_PORT_OVERRIDE + #define XMEMALIGN(a, s, h, t) ((void)(h), (void)(t), \ + wolfSSL_MemAlign((a), (s))) + #define XFREEALIGN(p, h, t) do { (void)(h); (void)(t); \ + wolfSSL_FreeAlign((p)); } while (0) + #endif #endif /* WOLFSSL_DEBUG_MEMORY */ #endif /* WOLFSSL_STATIC_MEMORY */ #endif @@ -950,12 +971,225 @@ enum { #define WC_NO_CONSTRUCTORS #endif +#if !defined(WOLFSSL_HAVE_POSIX_MEMALIGN) && !defined(WOLFSSL_NO_POSIX_MEMALIGN) + #if (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) || \ + (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 600)) + #define WOLFSSL_HAVE_POSIX_MEMALIGN + #endif +#endif + +/* Bottommost aligned-allocation primitives. One pair, one name; the + * implementation is selected at compile time: + * - posix_memalign() (native heap) when sensed available and the build + * permits native heap use. WC_XMEMALIGN_NATIVE is defined so upper + * layers (wolfSSL_MemAlign()) know the allocation bypasses + * XMALLOC/wolfSSL_Malloc() and can apply their own accounting. + * - otherwise, over-allocation via XMALLOC()/XFREE() with a word32 offset + * slot immediately below the returned pointer. heap and tag are honored, + * keeping this correct over static pools, user allocators, and exotic + * ports. + * Both branches share one alignment policy: alignments below sizeof(void *) + * are floored to it, larger non-powers-of-2 are rejected (NULL). */ +#if defined(WC_XMEMALIGN_PORT_OVERRIDE) + /* Use definitions supplied by earlier include (e.g. linuxkm_wc_port.h or user_settings.h) */ +#elif defined(WOLFSSL_NO_MALLOC) && !defined(WOLFSSL_STATIC_MEMORY) + static WC_INLINE void *wc_xmemalign(size_t alignment, size_t req_size, void *heap, int tag) { + (void)alignment; + (void)req_size; + (void)heap; + (void)tag; + return NULL; + } + static WC_INLINE void wc_xfreealign(void *ptr, void *heap, int tag) { + (void)ptr; + (void)heap; + (void)tag; + } +#elif !defined(WOLFSSL_NO_MALLOC) && !defined(WOLFSSL_STATIC_MEMORY) && \ + defined(WOLFSSL_HAVE_POSIX_MEMALIGN) + #define WC_XMEMALIGN_NATIVE + static WC_INLINE void *wc_xmemalign(size_t alignment, size_t size, void *heap, int tag) { + void *ret = NULL; + (void)heap; + (void)tag; + /* posix_memalign() EINVALs on alignments that are not power-of-2 + * multiples of sizeof(void *), and sanitizers escalate that to an + * abort. */ + if (alignment < sizeof(void *)) + alignment = sizeof(void *); + else if ((alignment & (alignment - 1U)) != 0) + return NULL; + if (posix_memalign(&ret, alignment, size) != 0) /* native heap */ + return NULL; + else + return ret; + } + #define wc_xfreealign(ptr, h, t) do { (void)(h); (void)(t); free(ptr); } while (0) /* native heap */ +#else + static WC_INLINE void *wc_xmemalign(size_t alignment, size_t req_size, void *heap, int tag) { + void *alloc; + char *ret; + size_t alloc_size; + + /* silently tolerate pathologically small alignments, but reject larger + * unaligned alignments. */ + if (alignment < sizeof(void *)) + alignment = sizeof(void *); + else if ((alignment & (alignment-1U)) != 0) + return NULL; + alloc_size = req_size + sizeof(word32) + alignment - 1U; + if (alloc_size < req_size) + return NULL; + alloc = XMALLOC(alloc_size, heap, tag); + if (alloc == NULL) + return NULL; + ret = (char *)alloc + sizeof(word32) + alignment - 1U; + ret = (char *)((wc_ptr_t)ret & ~((wc_ptr_t)alignment - 1U)); + *(word32 *)(void *)(ret - sizeof(word32)) = (word32)(ret - (char *)alloc); + return (void *)ret; + } + static WC_INLINE void wc_xfreealign(void *ptr, void *heap, int tag) { + (void)tag; + if (ptr != NULL) { + char *p = (char *)ptr; + XFREE(p - *(word32 *)(void *)(p - sizeof(word32)), heap, tag); + } + } +#endif + +#ifndef XMEMALIGN + #define XMEMALIGN(a, s, h, t) wc_xmemalign(a, s, h, t) +#endif +#ifndef XFREEALIGN + #define XFREEALIGN(p, h, t) wc_xfreealign(p, h, t) +#endif + +/* Reallocation with scrubbing and/or alignment guarantees, composed over the + * XREALLOC/XMALLOC/XFREE/XMEMALIGN/XFREEALIGN macro bindings so that port + * overrides, wolfSSL memory callbacks, and static pools are all honored. + * + * NOTA BENE: unlike XREALLOC(), these take the current allocated size as an + * explicit argument, ordered chronologically: old_size then new size. The + * _SCRUBBED variants ForceZero() the old allocation before releasing it, and + * take responsibility for the copy away from the underlying allocator, so no + * secret bytes are ever abandoned on a free list. On failure (NULL return) the + * original allocation is always intact and, for _SCRUBBED, unscrubbed -- + * matching the realloc() contract. Passing a size of 0 frees the allocation + * (scrubbed for _SCRUBBED) and returns NULL, matching the classic + * realloc-to-zero idiom. Never pass an XMEMALIGN() allocation to + * XREALLOC()/XREALLOC_SCRUBBED()/XFREE(), nor a plain allocation to the ALIGN + * variants: regular or ALIGN variants must be used consistently across all heap + * operations for a given allocation, due to the fallback strategy. */ +#ifndef XREALLOC_SCRUBBED + #define XREALLOC_SCRUBBED(p, old_size, size, h, t) \ + wc_xrealloc_scrubbed(p, old_size, size, h, t) +#endif +#ifndef XREALLOCALIGN + #define XREALLOCALIGN(p, align, old_size, size, h, t) \ + wc_xreallocalign(p, align, old_size, size, h, t, 0) +#endif +#ifndef XREALLOCALIGN_SCRUBBED + #define XREALLOCALIGN_SCRUBBED(p, align, old_size, size, h, t) \ + wc_xreallocalign(p, align, old_size, size, h, t, 1) +#endif + #if defined(WOLFSSL_SMALL_STACK) && defined(WC_NO_CONSTRUCTORS) #error WOLFSSL_SMALL_STACK requires constructors. #endif #include +/* AESNI requires alignment and ARMASM gains some performance from it. + * Xilinx RSA operations require alignment. + */ +#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_ARMASM) || \ + defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_AFALG_XILINX) || \ + defined(WOLFSSL_XILINX) + #ifndef WOLFSSL_USE_ALIGN + #define WOLFSSL_USE_ALIGN + #endif +#endif /* WOLFSSL_AESNI || WOLFSSL_ARMASM || USE_INTEL_SPEEDUP || \ + * WOLFSSL_AFALG_XILINX */ + +/* ARM C-only builds: if the toolchain reports that the target does NOT + * support unaligned access, force the alignment-safe code paths. This + * catches Cortex-M (ARMv6-M, and ARMv7-M/v8-M built with + * -mno-unaligned-access) without penalizing unaligned-capable cores + * such as Cortex-A and AArch64. __ARM_FEATURE_UNALIGNED is defined by + * GCC, Clang and armclang per the ARM ACLE when unaligned access is + * available. */ +#if defined(__arm__) && !defined(__ARM_FEATURE_UNALIGNED) + #ifndef WOLFSSL_USE_ALIGN + #define WOLFSSL_USE_ALIGN + #endif +#endif + +/* Helpers for memory alignment */ +#ifndef XALIGNED + #if defined(__GNUC__) || defined(__llvm__) || \ + defined(__IAR_SYSTEMS_ICC__) + #define XALIGNED(x) __attribute__ ( (aligned (x))) + #elif defined(__KEIL__) + #define XALIGNED(x) __align(x) + #elif defined(__WATCOMC__) /* && (_MSC_VER or !_MSC_VER) */ + /* No align available for Open Watcom V2, expansion comment needed: */ + #define XALIGNED(x) /* null expansion */ + #elif defined(_MSC_VER) + /* disable align warning, we want alignment ! */ + #pragma warning(disable: 4324) + #define XALIGNED(x) __declspec (align (x)) + #else + #define XALIGNED(x) /* null expansion */ + #endif +#endif + +/* Only use alignment in wolfSSL/wolfCrypt if WOLFSSL_USE_ALIGN is set */ +#ifdef WOLFSSL_USE_ALIGN + /* For IAR ARM the maximum variable alignment on stack is 8-bytes. + * Variables declared outside stack (like static globals) can have + * higher alignment. */ + #if defined(__ICCARM__) + #define WOLFSSL_ALIGN(x) XALIGNED(8) + #else + #define WOLFSSL_ALIGN(x) XALIGNED(x) + #endif +#else + #define WOLFSSL_ALIGN(x) /* null expansion */ +#endif + +#ifndef ALIGN4 + #define ALIGN4 WOLFSSL_ALIGN(4) +#endif +#ifndef ALIGN8 + #define ALIGN8 WOLFSSL_ALIGN(8) +#endif +#ifndef ALIGN16 + #define ALIGN16 WOLFSSL_ALIGN(16) +#endif +#ifndef ALIGN32 + #define ALIGN32 WOLFSSL_ALIGN(32) +#endif +#ifndef ALIGN64 + #define ALIGN64 WOLFSSL_ALIGN(64) +#endif +#ifndef ALIGN128 + #define ALIGN128 WOLFSSL_ALIGN(128) +#endif +#ifndef ALIGN256 + #define ALIGN256 WOLFSSL_ALIGN(256) +#endif + +/* Define the unaligned type modifier across different compilers */ +#if defined(__GNUC__) || defined(__clang__) + /* Works on GCC 2.0+ and all versions of Clang */ + #define MAYBE_UNALIGNED __attribute__((aligned(1))) +#elif defined(_MSC_VER) + /* MSVC handles unaligned reads via hardware or __unaligned keyword */ + #define MAYBE_UNALIGNED __unaligned +#else + #define MAYBE_UNALIGNED +#endif + /* declare/free variable handling for async and smallstack */ #ifndef WC_ALLOC_DO_ON_FAILURE #define WC_ALLOC_DO_ON_FAILURE() WC_DO_NOTHING @@ -988,6 +1222,28 @@ enum { break; \ } \ } +#define WC_ALLOC_HEAP_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ + for (idx##VAR_NAME=0; idx##VAR_NAME<(VAR_ITEMS); idx##VAR_NAME++) { \ + (VAR_NAME)[idx##VAR_NAME] = (VAR_TYPE*)XMEMALIGN(VAR_ALIGN, VAR_SIZE, (HEAP), \ + DYNAMIC_TYPE_TMP_BUFFER); \ + if ((VAR_NAME)[idx##VAR_NAME] == NULL) { \ + for (inner_idx_##VAR_NAME = 0; \ + inner_idx_##VAR_NAME < idx##VAR_NAME; \ + inner_idx_##VAR_NAME++) { \ + XFREEALIGN((VAR_NAME)[inner_idx_##VAR_NAME], (HEAP), \ + DYNAMIC_TYPE_TMP_BUFFER); \ + (VAR_NAME)[inner_idx_##VAR_NAME] = NULL; \ + } \ + for (inner_idx_##VAR_NAME = idx##VAR_NAME + 1; \ + inner_idx_##VAR_NAME < (VAR_ITEMS); \ + inner_idx_##VAR_NAME++) { \ + (VAR_NAME)[inner_idx_##VAR_NAME] = NULL; \ + } \ + idx##VAR_NAME = 0; \ + WC_ALLOC_DO_ON_FAILURE(); \ + break; \ + } \ + } #define WC_CALLOC_HEAP_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) \ do { \ WC_ALLOC_HEAP_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP); \ @@ -999,6 +1255,17 @@ enum { } \ } \ } while (0) +#define WC_CALLOC_HEAP_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ + do { \ + WC_ALLOC_HEAP_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP); \ + if (idx##VAR_NAME != 0) { \ + for (idx##VAR_NAME=0; \ + idx##VAR_NAME<(VAR_ITEMS); \ + idx##VAR_NAME++) { \ + XMEMSET((VAR_NAME)[idx##VAR_NAME], 0, VAR_SIZE); \ + } \ + } \ + } while (0) #define WC_HEAP_ARRAY_OK(VAR_NAME) (idx##VAR_NAME != 0) #define WC_FREE_HEAP_ARRAY(VAR_NAME, VAR_ITEMS, HEAP) \ if (WC_HEAP_ARRAY_OK(VAR_NAME)) { \ @@ -1007,11 +1274,27 @@ enum { } \ idx##VAR_NAME = 0; \ } +#define WC_FREE_HEAP_ARRAY_ALIGNED(VAR_NAME, VAR_ITEMS, HEAP) \ + if (WC_HEAP_ARRAY_OK(VAR_NAME)) { \ + for (idx##VAR_NAME=0; idx##VAR_NAME<(VAR_ITEMS); idx##VAR_NAME++) { \ + XFREEALIGN((VAR_NAME)[idx##VAR_NAME], (HEAP), DYNAMIC_TYPE_TMP_BUFFER); \ + } \ + idx##VAR_NAME = 0; \ + } #if defined(WOLFSSL_SMALL_STACK) #define WC_DECLARE_VAR_IS_HEAP_ALLOC #define WC_DECLARE_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) \ VAR_TYPE* VAR_NAME = NULL + + /* For _ALIGNED variants, the allocation *base* is VAR_ALIGN-aligned; + * VAR_NAME[i] for i > 0 is VAR_ALIGN-aligned only if sizeof(VAR_TYPE) is a + * multiple of VAR_ALIGN. sizeof is always a multiple of the type's own + * alignment, so this holds naturally unless VAR_ALIGN exceeds VAR_TYPE's + * intrinsic alignment. */ + #define WC_DECLARE_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ + VAR_TYPE* VAR_NAME = NULL + #define WC_VAR_OK(VAR_NAME) ((VAR_NAME) != NULL) #define WC_ALLOC_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) \ do { \ @@ -1021,6 +1304,16 @@ enum { WC_ALLOC_DO_ON_FAILURE(); \ } \ } while (0) + + /* NOTA BENE: _ALIGNED allocs must have matching _ALIGNED frees. */ + #define WC_ALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ + do { \ + (VAR_NAME) = (VAR_TYPE*)XMEMALIGN(VAR_ALIGN, sizeof(VAR_TYPE) * (VAR_SIZE), \ + (HEAP), DYNAMIC_TYPE_WOLF_BIGINT); \ + if ((VAR_NAME) == NULL) { \ + WC_ALLOC_DO_ON_FAILURE(); \ + } \ + } while (0) #define WC_ALLOC_VAR_EX(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP, TY, ONFAIL)\ do { \ (VAR_NAME) = (VAR_TYPE*)XMALLOC(sizeof(VAR_TYPE) * (VAR_SIZE), \ @@ -1029,6 +1322,17 @@ enum { ONFAIL; \ } \ } while (0) + + /* NOTA BENE: _ALIGNED allocs must have matching _ALIGNED frees. */ + #define WC_ALLOC_VAR_ALIGNED_EX(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP, TY, ONFAIL)\ + do { \ + (VAR_NAME) = (VAR_TYPE*)XMEMALIGN(VAR_ALIGN, sizeof(VAR_TYPE) * (VAR_SIZE), \ + (HEAP), TY); \ + if ((VAR_NAME) == NULL) { \ + ONFAIL; \ + } \ + } while (0) + #define WC_CALLOC_VAR_EX(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP, TY, ONFAIL)\ do { \ WC_ALLOC_VAR_EX(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP, TY, ONFAIL);\ @@ -1036,55 +1340,110 @@ enum { XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE) * (VAR_SIZE)); \ } \ } while (0) + + /* NOTA BENE: _ALIGNED allocs must have matching _ALIGNED frees. */ + #define WC_CALLOC_VAR_ALIGNED_EX(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP, TY, ONFAIL)\ + do { \ + WC_ALLOC_VAR_ALIGNED_EX(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP, TY, ONFAIL);\ + if ((VAR_NAME) != NULL) { \ + XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE) * (VAR_SIZE)); \ + } \ + } while (0) #define WC_CALLOC_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) \ do { \ WC_ALLOC_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP); \ XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE) * (VAR_SIZE)); \ } while (0) + + /* NOTA BENE: _ALIGNED allocs must have matching _ALIGNED frees. */ + #define WC_CALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ + do { \ + WC_ALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP); \ + XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE) * (VAR_SIZE)); \ + } while (0) + #define WC_FREE_VAR(VAR_NAME, HEAP) \ XFREE(VAR_NAME, (HEAP), DYNAMIC_TYPE_WOLF_BIGINT) + #define WC_FREE_VAR_ALIGNED(VAR_NAME, HEAP) \ + XFREEALIGN(VAR_NAME, (HEAP), DYNAMIC_TYPE_WOLF_BIGINT) #define WC_FREE_VAR_EX(VAR_NAME, HEAP, TYPE) \ XFREE(VAR_NAME, (HEAP), TYPE) + #define WC_FREE_VAR_ALIGNED_EX(VAR_NAME, HEAP, TYPE) \ + XFREEALIGN(VAR_NAME, (HEAP), TYPE) #define WC_DECLARE_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) \ + wc_static_assert(((VAR_SIZE) % sizeof(VAR_TYPE)) == 0); \ + WC_DECLARE_HEAP_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) + #define WC_DECLARE_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ + wc_static_assert(((VAR_SIZE) % sizeof(VAR_TYPE)) == 0); \ + wc_static_assert(((VAR_SIZE) % (VAR_ALIGN)) == 0); \ WC_DECLARE_HEAP_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) #define WC_ARRAY_ARG(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE) \ WC_HEAP_ARRAY_ARG(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE) #define WC_ALLOC_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) \ WC_ALLOC_HEAP_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) + #define WC_ALLOC_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ + WC_ALLOC_HEAP_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) #define WC_CALLOC_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) \ WC_CALLOC_HEAP_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) + #define WC_CALLOC_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ + WC_CALLOC_HEAP_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) #define WC_ARRAY_OK(VAR_NAME) WC_HEAP_ARRAY_OK(VAR_NAME) #define WC_FREE_ARRAY(VAR_NAME, VAR_ITEMS, HEAP) \ WC_FREE_HEAP_ARRAY(VAR_NAME, VAR_ITEMS, HEAP) + #define WC_FREE_ARRAY_ALIGNED(VAR_NAME, VAR_ITEMS, HEAP) \ + WC_FREE_HEAP_ARRAY_ALIGNED(VAR_NAME, VAR_ITEMS, HEAP) #else #undef WC_DECLARE_VAR_IS_HEAP_ALLOC #define WC_DECLARE_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) \ VAR_TYPE VAR_NAME[VAR_SIZE] + #define WC_DECLARE_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ + XALIGNED(VAR_ALIGN) VAR_TYPE VAR_NAME[VAR_SIZE] #define WC_ALLOC_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) WC_DO_NOTHING + #define WC_ALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) WC_DO_NOTHING #define WC_ALLOC_VAR_EX(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP, TYPE, ONFAIL)\ WC_DO_NOTHING + #define WC_ALLOC_VAR_ALIGNED_EX(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP, TYPE, ONFAIL)\ + WC_DO_NOTHING #define WC_VAR_OK(VAR_NAME) 1 #define WC_CALLOC_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) \ XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE)) + #define WC_CALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ + XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE)) #define WC_CALLOC_VAR_EX(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP, TY, ONFAIL)\ XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE)) + #define WC_CALLOC_VAR_ALIGNED_EX(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP, TY, ONFAIL)\ + XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE)) #define WC_FREE_VAR(VAR_NAME, HEAP) WC_DO_NOTHING \ - /* nothing to free, its stack */ + /* nothing to free, it's stack */ + #define WC_FREE_VAR_ALIGNED(VAR_NAME, HEAP) WC_DO_NOTHING \ + /* nothing to free, it's stack */ #define WC_FREE_VAR_EX(VAR_NAME, HEAP, TYPE) WC_DO_NOTHING + #define WC_FREE_VAR_ALIGNED_EX(VAR_NAME, HEAP, TYPE) WC_DO_NOTHING #define WC_DECLARE_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) \ + wc_static_assert(((VAR_SIZE) % sizeof(VAR_TYPE)) == 0); \ VAR_TYPE VAR_NAME[VAR_ITEMS][(VAR_SIZE) / sizeof(VAR_TYPE)] /* NOLINT(bugprone-sizeof-expression) */ + #define WC_DECLARE_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ + wc_static_assert(((VAR_SIZE) % sizeof(VAR_TYPE)) == 0); \ + wc_static_assert(((VAR_SIZE) % (VAR_ALIGN)) == 0); \ + XALIGNED(VAR_ALIGN) VAR_TYPE VAR_NAME[VAR_ITEMS][(VAR_SIZE) / sizeof(VAR_TYPE)] /* NOLINT(bugprone-sizeof-expression) */ #define WC_ARRAY_ARG(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE) \ VAR_TYPE VAR_NAME[VAR_ITEMS][(VAR_SIZE) / sizeof(VAR_TYPE)] /* NOLINT(bugprone-sizeof-expression) */ #define WC_ALLOC_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) \ WC_DO_NOTHING + #define WC_ALLOC_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ + WC_DO_NOTHING #define WC_CALLOC_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) \ XMEMSET(VAR_NAME, 0, sizeof(VAR_NAME)) + #define WC_CALLOC_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ + XMEMSET(VAR_NAME, 0, sizeof(VAR_NAME)) #define WC_ARRAY_OK(VAR_NAME) 1 #define WC_FREE_ARRAY(VAR_NAME, VAR_ITEMS, HEAP) WC_DO_NOTHING \ /* nothing to free, its stack */ + #define WC_FREE_ARRAY_ALIGNED(VAR_NAME, VAR_ITEMS, HEAP) WC_DO_NOTHING \ + /* nothing to free, its stack */ #endif -#if defined(HAVE_FIPS) || defined(HAVE_SELFTEST) +#if (defined(HAVE_FIPS) && FIPS_VERSION3_LT(7,0,0)) || defined(HAVE_SELFTEST) /* These are here for the FIPS code that can't be changed. * New definitions don't need to be added here. */ #define DECLARE_VAR WC_DECLARE_VAR @@ -1120,7 +1479,6 @@ enum { #define XMEMSET(b,c,l) memset((b),(c),(l)) #define XMEMCMP(s1,s2,n) memcmp((s1),(s2),(n)) #define XMEMMOVE(d,s,l) memmove((d),(s),(l)) - #define XSTRLEN(s1) strlen((s1)) #define XSTRNCPY(s1,s2,n) strncpy((s1),(s2),(n)) /* strstr, strncmp, strcmp, and strncat only used by wolfSSL proper, @@ -1951,97 +2309,6 @@ WOLFSSL_API word32 CheckRunTimeSettings(void); #define XASM_LINK(f) asm(f) #endif -/* AESNI requires alignment and ARMASM gains some performance from it. - * Xilinx RSA operations require alignment. - */ -#if defined(WOLFSSL_AESNI) || defined(WOLFSSL_ARMASM) || \ - defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_AFALG_XILINX) || \ - defined(WOLFSSL_XILINX) - #ifndef WOLFSSL_USE_ALIGN - #define WOLFSSL_USE_ALIGN - #endif -#endif /* WOLFSSL_AESNI || WOLFSSL_ARMASM || USE_INTEL_SPEEDUP || \ - * WOLFSSL_AFALG_XILINX */ - -/* ARM C-only builds: if the toolchain reports that the target does NOT - * support unaligned access, force the alignment-safe code paths. This - * catches Cortex-M (ARMv6-M, and ARMv7-M/v8-M built with - * -mno-unaligned-access) without penalizing unaligned-capable cores - * such as Cortex-A and AArch64. __ARM_FEATURE_UNALIGNED is defined by - * GCC, Clang and armclang per the ARM ACLE when unaligned access is - * available. */ -#if defined(__arm__) && !defined(__ARM_FEATURE_UNALIGNED) - #ifndef WOLFSSL_USE_ALIGN - #define WOLFSSL_USE_ALIGN - #endif -#endif - -/* Helpers for memory alignment */ -#ifndef XALIGNED - #if defined(__GNUC__) || defined(__llvm__) || \ - defined(__IAR_SYSTEMS_ICC__) - #define XALIGNED(x) __attribute__ ( (aligned (x))) - #elif defined(__KEIL__) - #define XALIGNED(x) __align(x) - #elif defined(__WATCOMC__) /* && (_MSC_VER or !_MSC_VER) */ - /* No align available for Open Watcom V2, expansion comment needed: */ - #define XALIGNED(x) /* null expansion */ - #elif defined(_MSC_VER) - /* disable align warning, we want alignment ! */ - #pragma warning(disable: 4324) - #define XALIGNED(x) __declspec (align (x)) - #else - #define XALIGNED(x) /* null expansion */ - #endif -#endif - -/* Only use alignment in wolfSSL/wolfCrypt if WOLFSSL_USE_ALIGN is set */ -#ifdef WOLFSSL_USE_ALIGN - /* For IAR ARM the maximum variable alignment on stack is 8-bytes. - * Variables declared outside stack (like static globals) can have - * higher alignment. */ - #if defined(__ICCARM__) - #define WOLFSSL_ALIGN(x) XALIGNED(8) - #else - #define WOLFSSL_ALIGN(x) XALIGNED(x) - #endif -#else - #define WOLFSSL_ALIGN(x) /* null expansion */ -#endif - -#ifndef ALIGN4 - #define ALIGN4 WOLFSSL_ALIGN(4) -#endif -#ifndef ALIGN8 - #define ALIGN8 WOLFSSL_ALIGN(8) -#endif -#ifndef ALIGN16 - #define ALIGN16 WOLFSSL_ALIGN(16) -#endif -#ifndef ALIGN32 - #define ALIGN32 WOLFSSL_ALIGN(32) -#endif -#ifndef ALIGN64 - #define ALIGN64 WOLFSSL_ALIGN(64) -#endif -#ifndef ALIGN128 - #define ALIGN128 WOLFSSL_ALIGN(128) -#endif -#ifndef ALIGN256 - #define ALIGN256 WOLFSSL_ALIGN(256) -#endif - -/* Define the unaligned type modifier across different compilers */ -#if defined(__GNUC__) || defined(__clang__) - /* Works on GCC 2.0+ and all versions of Clang */ - #define MAYBE_UNALIGNED __attribute__((aligned(1))) -#elif defined(_MSC_VER) - /* MSVC handles unaligned reads via hardware or __unaligned keyword */ - #define MAYBE_UNALIGNED __unaligned -#else - #define MAYBE_UNALIGNED -#endif - #if !defined(PEDANTIC_EXTENSION) #if defined(__GNUC__) #define PEDANTIC_EXTENSION __extension__ From 343c74bbd62634ddfa8c52d7b8968c68ac459e7a Mon Sep 17 00:00:00 2001 From: Aidan Keefe Date: Wed, 28 Oct 2026 16:11:21 -0600 Subject: [PATCH 2/5] aligned alloc rework --- .wolfssl_known_macro_extras | 2 - linuxkm/linuxkm_wc_port.h | 19 +-- wolfcrypt/src/memory.c | 227 ++++++++++++++++++++++-------------- wolfcrypt/test/test.c | 117 +++++++++++-------- wolfssl/wolfcrypt/memory.h | 15 ++- wolfssl/wolfcrypt/types.h | 127 +++++--------------- 6 files changed, 255 insertions(+), 252 deletions(-) diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 83af5346fe0..02a88c29308 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -1110,7 +1110,6 @@ WOLFSSL_NO_KTRI_ORACLE_WARNING WOLFSSL_NO_LMS_SHAKE256_256 WOLFSSL_NO_OCSP_DATE_CHECK WOLFSSL_NO_OCSP_OPTIONAL_CERTS -WOLFSSL_NO_POSIX_MEMALIGN WOLFSSL_NO_RSA_KEY_CHECK WOLFSSL_NO_RSA_SW WOLFSSL_NO_SERVER_GROUPS_EXT @@ -1349,7 +1348,6 @@ _UINTPTR_T_DECLARED _WIN32 _WIN32_WCE _WIN64 -_XOPEN_SOURCE _XOPEN_SOURCE_EXTENDED __32MZ2048ECH144__ __32MZ2048ECM144__ diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 16dd555da4d..124979f90b5 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -1,4 +1,4 @@ -/* linuxkm_wc_port.h +/* linuxkm_wc_port.hlunuc * * Copyright (C) 2006-2026 wolfSSL Inc. * @@ -2419,12 +2419,17 @@ #if defined(USE_KVREALLOC) || !defined(USE_KVMALLOC) #define XREALLOC(p, n, h, t) __extension__ ({(void)(h); (void)(t); realloc(p, n);}) #endif - /* The default linuxkm malloc wrappers (see above) assure adequate - * alignment. In particular we do not need to worry about alignments - * larger than the allocation size, so rounded-up-size-based alignment - * is enough. */ - #define XMEMALIGN(a, s, h, t) ((void)(a), (void)(h), (void)(t), malloc(s)) - #define XFREEALIGN(p, h, t) do { (void)(h); (void)(t); free(p); } while (0) +/* Since linux commit 59bb47985c (5.4), kmalloc with power-of-2 size is + * aligned to the size, and malloc() rounds up via WC_LINUXKM_ROUND_UP_P_OF_2, + * so allocating max(a, s) suffices. Older kernels use the wc_xmemalign() + * shim in memory.c. */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) + #define XMEMALIGN(a, s, h, t) __extension__ ({ \ + size_t _wc_a = (a), _wc_s = (s); \ + (void)(h); (void)(t); \ + malloc((_wc_s < _wc_a) ? _wc_a : _wc_s); }) + #define XFREEALIGN(p, h, t) do { (void)(h); (void)(t); free(p); } while (0) +#endif #endif /* !XMALLOC_USER && !XMALLOC_OVERRIDE */ #endif diff --git a/wolfcrypt/src/memory.c b/wolfcrypt/src/memory.c index 27ca3a34fa5..c3ce00d4714 100644 --- a/wolfcrypt/src/memory.c +++ b/wolfcrypt/src/memory.c @@ -79,6 +79,61 @@ void *z_realloc(void *ptr, size_t size) #define realloc z_realloc #endif +/* Alignment policy shared by the aligned allocation and reallocation paths: + * alignments below the floor (the larger of sizeof(void *) and the word32 + * offset slot) are raised to it, and larger non-powers-of-2 are rejected. + * Returns the effective alignment, or 0 if rejected. */ +#define WC_ALIGN_FLOOR (sizeof(void *) > sizeof(word32) ? \ + sizeof(void*) : sizeof(word32)) +static WC_INLINE size_t wc_xmemalign_floor(size_t alignment) +{ + if (alignment < WC_ALIGN_FLOOR) + return WC_ALIGN_FLOOR; + if ((alignment & (alignment - 1U)) != 0) + return 0; + return alignment; +} + +#ifndef WC_XMEMALIGN_PORT_OVERRIDE +/* Aligned-allocation shim layout, shared by wc_xmemalign()/wc_xfreealign() + * and wolfSSL_MemAlign()/wolfSSL_FreeAlign(): over-allocate, align within + * the block, and record the offset to its start in a word32 slot immediately + * below the returned pointer. + * + * Size of the underlying allocation for an aligned request, or 0 if the + * alignment is rejected or the padded size wraps. *alignment is floored + * in place per wc_xmemalign_floor(). */ +static WC_INLINE size_t wc_xmemalign_alloc_size(size_t *alignment, + size_t req_size) +{ + size_t alloc_size; + + *alignment = wc_xmemalign_floor(*alignment); + if (*alignment == 0) + return 0; + alloc_size = req_size + sizeof(word32) + *alignment - 1U; + if (alloc_size < req_size) + return 0; + return alloc_size; +} + +/* Aligned pointer within alloc, with its offset recorded just below. */ +static WC_INLINE void *wc_xmemalign_place(void *alloc, size_t alignment) +{ + char *ret = (char *)alloc + sizeof(word32) + alignment - 1U; + ret = (char *)((wc_ptr_t)ret & ~((wc_ptr_t)alignment - 1U)); + *(word32 *)(void *)(ret - sizeof(word32)) = (word32)(ret - (char *)alloc); + return (void *)ret; +} + +/* Underlying allocation for an aligned pointer (non-NULL). */ +static WC_INLINE void *wc_xmemalign_base(void *ptr) +{ + char *p = (char *)ptr; + return (void *)(p - *(word32 *)(void *)(p - sizeof(word32))); +} +#endif /* !WC_XMEMALIGN_PORT_OVERRIDE */ + #ifdef USE_WOLFSSL_MEMORY #include @@ -486,68 +541,43 @@ void wolfSSL_Free(void *ptr) } } -#ifndef WC_XMEMALIGN_PORT_OVERRIDE -/* Aligned allocation wrappers over the wc_xmemalign()/wc_xfreealign() - * bottommost primitives. When the primitives' over-allocation branch is in - * effect, the underlying memory comes from XMALLOC/XFREE, i.e. - * wolfSSL_Malloc()/wolfSSL_Free() here, so allocator callbacks and all - * allocation accounting apply automatically. When the native - * posix_memalign() branch is in effect (WC_XMEMALIGN_NATIVE), the allocation - * bypasses wolfSSL_Malloc(): WOLFSSL_MEM_FAIL_COUNT accounting is applied - * here instead, while registered allocator callbacks and - * WOLFSSL_CHECK_MEM_ZERO coverage deliberately do not extend to aligned - * allocations in that configuration. */ -#ifdef WOLFSSL_DEBUG_MEMORY +#if !defined(WC_XMEMALIGN_PORT_OVERRIDE) && defined(WOLFSSL_DEBUG_MEMORY) +/* Aligned allocation wrappers using the wc_xmemalign() shim layout. The + * underlying memory comes from wolfSSL_Malloc()/wolfSSL_Free(), called here + * directly so that the caller's func/line reach allocator callbacks (e.g. + * WOLFSSL_TRACK_MEMORY) instead of this file's. Non-debug builds use + * wc_xmemalign()/wc_xfreealign(), which reach wolfSSL_Malloc()/wolfSSL_Free() + * through XMALLOC()/XFREE() with identical effect. */ void* wolfSSL_MemAlign(size_t alignment, size_t size, const char* func, unsigned int line) -#else -void* wolfSSL_MemAlign(size_t alignment, size_t size) -#endif { - void* res; + void* res = NULL; + void* alloc; + size_t alloc_size = wc_xmemalign_alloc_size(&alignment, size); -#if defined(WC_XMEMALIGN_NATIVE) && defined(WOLFSSL_MEM_FAIL_COUNT) - if (!wc_MemFailCount_AllocMem()) { - WOLFSSL_MSG("MemFailCnt: Fail memalign"); - return NULL; + if (alloc_size != 0) { + alloc = wolfSSL_Malloc(alloc_size, func, line); + if (alloc != NULL) + res = wc_xmemalign_place(alloc, alignment); } -#endif - res = wc_xmemalign(alignment, size, NULL, DYNAMIC_TYPE_TMP_BUFFER); -#ifdef WOLFSSL_DEBUG_MEMORY #if defined(WOLFSSL_DEBUG_MEMORY_PRINT) && !defined(WOLFSSL_TRACK_MEMORY) fprintf(stderr, "MemAlign: %p -> %u (align %u) at %s:%u\n", res, (word32)size, (word32)alignment, func, line); -#else - (void)func; - (void)line; -#endif #endif return res; } -#ifdef WOLFSSL_DEBUG_MEMORY void wolfSSL_FreeAlign(void *ptr, const char* func, unsigned int line) -#else -void wolfSSL_FreeAlign(void *ptr) -#endif { -#ifdef WOLFSSL_DEBUG_MEMORY #if defined(WOLFSSL_DEBUG_MEMORY_PRINT) && !defined(WOLFSSL_TRACK_MEMORY) fprintf(stderr, "FreeAlign: %p at %s:%u\n", ptr, func, line); -#else - (void)func; - (void)line; -#endif #endif -#if defined(WC_XMEMALIGN_NATIVE) && defined(WOLFSSL_MEM_FAIL_COUNT) if (ptr != NULL) - wc_MemFailCount_FreeMem(); -#endif - wc_xfreealign(ptr, NULL, DYNAMIC_TYPE_TMP_BUFFER); + wolfSSL_Free(wc_xmemalign_base(ptr), func, line); } -#endif /* !WC_XMEMALIGN_PORT_OVERRIDE */ +#endif /* !WC_XMEMALIGN_PORT_OVERRIDE && WOLFSSL_DEBUG_MEMORY */ #ifdef WOLFSSL_DEBUG_MEMORY void* wolfSSL_Realloc(void *ptr, size_t size, const char* func, unsigned int line) @@ -1800,6 +1830,50 @@ void wc_ForceZero(void *mem, size_t len) } #endif +#ifndef WC_XMEMALIGN_PORT_OVERRIDE +/* Default XMEMALIGN()/XFREEALIGN(): the shim layout over XMALLOC()/XFREE(), + * so heap and tag are honored, keeping this correct over static pools, user + * allocators, allocator callbacks, and exotic ports. Alignment policy per + * wc_xmemalign_floor(). */ +void *wc_xmemalign(size_t alignment, size_t size, void *heap, int tag) +{ + void *alloc; + size_t alloc_size = wc_xmemalign_alloc_size(&alignment, size); + + (void)heap; + (void)tag; + + if (alloc_size == 0) + return NULL; + alloc = XMALLOC(alloc_size, heap, tag); + if (alloc == NULL) + return NULL; + return wc_xmemalign_place(alloc, alignment); +} + +void wc_xfreealign(void *ptr, void *heap, int tag) +{ + (void)heap; + (void)tag; + + if (ptr != NULL) + XFREE(wc_xmemalign_base(ptr), heap, tag); +} +#endif + +static void wc_xrealloc_scrub(void *ptr, size_t len) +{ +#ifdef WOLFSSL_NO_FORCE_ZERO + /* volatile stores: a plain XMEMSET() just ahead of XFREE() is a dead + * store the compiler may remove. */ + volatile byte *p = (volatile byte *)ptr; + while (len-- > 0) + *p++ = 0; +#else + ForceZero(ptr, len); +#endif +} + /* Reallocation with scrubbing and/or alignment guarantees: implementations * behind the XREALLOC_SCRUBBED()/XREALLOCALIGN()/XREALLOCALIGN_SCRUBBED() * macros (types.h), composed over the XREALLOC/XMALLOC/XFREE/XMEMALIGN/ @@ -1815,30 +1889,25 @@ void *wc_xrealloc_scrubbed(void *ptr, size_t old_size, size_t size, (void)heap; (void)tag; - if (ptr == NULL) + if (ptr == NULL && size != 0) return XMALLOC(size, heap, tag); - if (size == 0) { + else if (ptr != NULL && size == 0) { /* realloc-to-zero idiom: scrub, free, and return NULL. Handled * explicitly (and ahead of all other arms) so the semantics are * uniform across builds rather than the underlying allocator's * implementation-defined choice. */ -#ifdef WOLFSSL_NO_FORCE_ZERO - XMEMSET(ptr, 0, old_size); -#else - ForceZero(ptr, old_size); -#endif + wc_xrealloc_scrub(ptr, old_size); XFREE(ptr, heap, tag); return NULL; } + else if (ptr == NULL && size == 0) { + return NULL; + } new_p = XMALLOC(size, heap, tag); if (new_p == NULL) return NULL; /* original intact and unscrubbed */ XMEMCPY(new_p, ptr, (old_size < size) ? old_size : size); -#ifdef WOLFSSL_NO_FORCE_ZERO - XMEMSET(ptr, 0, old_size); -#else - ForceZero(ptr, old_size); -#endif + wc_xrealloc_scrub(ptr, old_size); XFREE(ptr, heap, tag); return new_p; } @@ -1851,17 +1920,10 @@ void *wc_xreallocalign(void *ptr, size_t alignment, size_t old_size, (void)heap; if (size == 0) { /* realloc-to-zero idiom: free (scrubbed if requested) and return - * NULL. Handled explicitly, ahead of the delegation and retention - * arms: delegation would inherit the underlying allocator's - * implementation-defined choice, and retention would wrongly keep a - * small allocation alive. */ - if (scrub) { -#ifdef WOLFSSL_NO_FORCE_ZERO - XMEMSET(ptr, 0, old_size); -#else - ForceZero(ptr, old_size); -#endif - } + * NULL. Handled explicitly, ahead of the retention arm, which would + * wrongly keep a small allocation alive. */ + if (scrub && (ptr != NULL)) + wc_xrealloc_scrub(ptr, old_size); XFREEALIGN(ptr, heap, tag); return NULL; } @@ -1872,44 +1934,31 @@ void *wc_xreallocalign(void *ptr, size_t alignment, size_t old_size, return XMEMALIGN(alignment, size, heap, tag); /* Match the alignment policy of the allocation-side primitives, so the - * shrink-retention and delegation tests below reason about the alignment - * actually in effect. */ - if (alignment < sizeof(void *)) - alignment = sizeof(void *); - else if ((alignment & (alignment - 1U)) != 0) + * shrink-retention test below reasons about the alignment actually in + * effect. */ + alignment = wc_xmemalign_floor(alignment); + if (alignment == 0) return NULL; /* original intact */ -#if defined(WC_XMEMALIGN_NATIVE) - /* posix_memalign() memory is first-class native-heap memory, and native - * realloc() results are guaranteed at least sizeof(void *) alignment, so - * requests at or below that can delegate wholesale. This arm is - * unavailable when XMEMALIGN() is the over-allocation shim (the pointer - * is interior to the underlying allocation) or a port override (unknown - * provenance). */ - if ((!scrub) && (alignment == sizeof(void *))) - return XREALLOC(ptr, size, heap, tag); -#endif - /* Shrink retention: when the reclaimable slack cannot exceed the aligned * allocator's own overhead, compaction is footprint-neutral at best -- * retain the allocation. Larger shrinks fall through and genuinely - * release memory. (Nothing is released here, so no scrub is owed.) */ + * release memory. The caller forgets the tail [size, old_size), and + * later scrubs cover only size bytes, so scrub the tail now. */ if ((size <= old_size) && ((old_size - size) <= ((alignment - 1U) + sizeof(word32))) && - (((wc_ptr_t)ptr & ((wc_ptr_t)alignment - 1U)) == 0)) + (((wc_ptr_t)ptr & ((wc_ptr_t)alignment - 1U)) == 0)) { + if (scrub && (old_size > size)) + wc_xrealloc_scrub((byte *)ptr + size, old_size - size); return ptr; + } new_p = XMEMALIGN(alignment, size, heap, tag); if (new_p == NULL) return NULL; /* original intact and unscrubbed */ XMEMCPY(new_p, ptr, (old_size < size) ? old_size : size); - if (scrub) { -#ifdef WOLFSSL_NO_FORCE_ZERO - XMEMSET(ptr, 0, old_size); -#else - ForceZero(ptr, old_size); -#endif - } + if (scrub) + wc_xrealloc_scrub(ptr, old_size); XFREEALIGN(ptr, heap, tag); return new_p; } diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index e733dbe009c..0ae023f6b26 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -26592,25 +26592,27 @@ typedef struct keywrapVector { #if !defined(HAVE_FIPS) && !defined(HAVE_SELFTEST) && \ !defined(WOLFSSL_NO_MALLOC) +/* Heap, not stack: GCM tables make struct Aes too big for the frame limit. + * Aligned: armv8-a+crypto NEON stores fault on an under-aligned Aes. */ #define AESKEYWRAP_AES_ALIGN 16 static wc_test_ret_t aeskeywrap_caller_aes_test(const keywrapVector* v, byte* output, word32 outputSz, byte* plain, word32 plainBufSz) { - WC_DECLARE_VAR_ALIGNED(aes, Aes, AESKEYWRAP_AES_ALIGN, 1, HEAP_HINT); + Aes* aes; wc_test_ret_t ret = 0; int wrapSz = 0, plainSz; - /* Alignment here is semantic, not advisory: armv8-a+crypto uses - * alignment-qualified NEON stores that fault on an under-aligned Aes. - * XMALLOC guarantees only 8. */ - WC_ALLOC_VAR_ALIGNED_EX(aes, Aes, AESKEYWRAP_AES_ALIGN, 1, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER, return WC_TEST_RET_ENC_EC(MEMORY_E)); + aes = (Aes*)XMEMALIGN(AESKEYWRAP_AES_ALIGN, sizeof(Aes), HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (aes == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); XMEMSET(output, 0, outputSz); XMEMSET(plain, 0, plainBufSz); if (wc_AesInit(aes, HEAP_HINT, devId) != 0) { - WC_FREE_VAR_ALIGNED_EX(aes, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + XFREEALIGN(aes, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); return WC_TEST_RET_ENC_NC; } @@ -26621,7 +26623,8 @@ static wc_test_ret_t aeskeywrap_caller_aes_test(const keywrapVector* v, NULL); if ((wrapSz < 0) || (wrapSz != (int)v->verifyLen) || XMEMCMP(output, v->verify, v->verifyLen) != 0) { - ret = WC_TEST_RET_ENC_EC(wrapSz); + ret = (wrapSz < 0) ? WC_TEST_RET_ENC_EC(wrapSz) : + WC_TEST_RET_ENC_NC; } } if (ret == 0) { @@ -26633,12 +26636,13 @@ static wc_test_ret_t aeskeywrap_caller_aes_test(const keywrapVector* v, plainBufSz, NULL); if ((plainSz < 0) || (plainSz != (int)v->dataLen) || XMEMCMP(plain, v->data, v->dataLen) != 0) { - ret = WC_TEST_RET_ENC_EC(plainSz); + ret = (plainSz < 0) ? WC_TEST_RET_ENC_EC(plainSz) : + WC_TEST_RET_ENC_NC; } } wc_AesFree(aes); - WC_FREE_VAR_ALIGNED_EX(aes, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + XFREEALIGN(aes, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); return ret; } #endif @@ -37900,7 +37904,12 @@ static wc_test_ret_t simple_memalign_test(size_t alignment, size_t sz) return ret; } -#define WC_TEST_MA_ALIGN 64U +#if defined(__ICCARM__) && !defined(WC_DECLARE_VAR_IS_HEAP_ALLOC) + /* IAR ARM caps stack alignment at 8 (asserted by WC_DECLARE_*_ALIGNED) */ + #define WC_TEST_MA_ALIGN 8 +#else + #define WC_TEST_MA_ALIGN 64 +#endif #define WC_TEST_MA_SZ 256U /* multiple of WC_TEST_MA_ALIGN: see the stride * asserts in WC_DECLARE_ARRAY_ALIGNED. */ static wc_test_ret_t memalign_macros_test(void) @@ -37924,8 +37933,10 @@ static wc_test_ret_t memalign_macros_test(void) /* XALIGNED() expands to nothing on compilers with no known alignment * attribute, so only assert alignment where the declaration (stack * build) or XMEMALIGN() (heap build) actually guarantees it. */ -#if defined(WC_DECLARE_VAR_IS_HEAP_ALLOC) || defined(__GNUC__) || \ - defined(__CC_ARM) || defined(_MSC_VER) +#if defined(WC_DECLARE_VAR_IS_HEAP_ALLOC) || \ + (!defined(__WATCOMC__) && \ + (defined(__GNUC__) || defined(__llvm__) || \ + defined(__IAR_SYSTEMS_ICC__) || defined(__KEIL__) || defined(_MSC_VER))) if ((ret == 0) && (((wc_ptr_t)vbuf & (WC_TEST_MA_ALIGN - 1U)) != 0)) { ret = WC_TEST_RET_ENC_NC; } @@ -37964,21 +37975,13 @@ static wc_test_ret_t memalign_macros_test(void) ret = WC_TEST_RET_ENC_EC(MEMORY_E)); } if (ret == 0) { -#ifdef WC_DECLARE_VAR_IS_HEAP_ALLOC - /* heap arm zeroes the full extent. */ + /* both arms zero the full extent. */ for (i = 0; i < (int)WC_TEST_MA_SZ; i++) { if (vbuf[i] != 0) { ret = WC_TEST_RET_ENC_I((word32)i); break; } } -#else - /* stack arm's CALLOC zeroes only sizeof(VAR_TYPE) (pre-existing - * behavior mirrored from WC_CALLOC_VAR): only [0] is guaranteed. */ - if (vbuf[0] != 0) { - ret = WC_TEST_RET_ENC_NC; - } -#endif } WC_FREE_VAR_ALIGNED_EX(vbuf, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); @@ -38013,39 +38016,40 @@ static wc_test_ret_t memalign_realloc_test(void) XFREEALIGN(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); return WC_TEST_RET_ENC_EC(MEMORY_E); } - if (((wc_ptr_t)q & 63U) != 0) + /* From here on, p always holds the live allocation: a successful realloc + * has released the old pointer, and a failed one leaves it intact. */ + p = q; + if (((wc_ptr_t)p & 63U) != 0) ret = WC_TEST_RET_ENC_NC; for (i = 0; (ret == 0) && (i < 300); i++) { - if (q[i] != (byte)i) + if (p[i] != (byte)i) ret = WC_TEST_RET_ENC_I((word32)i); } - /* small shrink: retention (pointer identity), reclaimable slack below - * the aligned allocator's own overhead. */ + /* non-power-of-2 alignment: rejected, original intact. */ if (ret == 0) { - p = (byte*)XREALLOCALIGN(q, 64U, 1600U, 1590U, HEAP_HINT, - DYNAMIC_TYPE_TMP_BUFFER); - if (p != q) + if (XREALLOCALIGN(p, 24U, 1580U, 1600U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER) != NULL) ret = WC_TEST_RET_ENC_NC; - else - q = p; + for (i = 0; (ret == 0) && (i < 300); i++) { + if (p[i] != (byte)i) + ret = WC_TEST_RET_ENC_I((word32)i); + } } /* large shrink: compaction (may move); contents and alignment * preserved. */ if (ret == 0) { - p = (byte*)XREALLOCALIGN(q, 64U, 1600U, 128U, HEAP_HINT, + q = (byte*)XREALLOCALIGN(p, 64U, 1580U, 128U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); - if (p == NULL) { - ret = WC_TEST_RET_ENC_EC(MEMORY_E); - p = q; /* original intact on failure */ - } + if (q == NULL) + ret = WC_TEST_RET_ENC_EC(MEMORY_E); /* p intact on failure */ else { - q = p; - if (((wc_ptr_t)q & 63U) != 0) + p = q; + if (((wc_ptr_t)p & 63U) != 0) ret = WC_TEST_RET_ENC_NC; for (i = 0; (ret == 0) && (i < 128); i++) { - if (q[i] != (byte)i) + if (p[i] != (byte)i) ret = WC_TEST_RET_ENC_I((word32)i); } } @@ -38070,7 +38074,23 @@ static wc_test_ret_t memalign_realloc_test(void) if (q[i] != (byte)(255 - i)) ret = WC_TEST_RET_ENC_I((word32)i); } - XFREE(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (ret != 0) { + XFREE(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return ret; + } + + /* unaligned scrubbed shrink: leading contents preserved. */ + p = (byte*)XREALLOC_SCRUBBED(q, 400U, 50U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) { + XFREE(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_EC(MEMORY_E); + } + for (i = 0; (ret == 0) && (i < 50); i++) { + if (p[i] != (byte)(255 - i)) + ret = WC_TEST_RET_ENC_I((word32)i); + } + XFREE(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); /* realloc-to-zero idiom: frees and returns NULL. */ if (ret == 0) { @@ -38256,9 +38276,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memory_test(void) #endif #if defined(WOLFSSL_STATIC_MEMORY) || !defined(WOLFSSL_NO_MALLOC) - /* aligned allocation roundtrips. Every allocation is kept at least as - * large as its alignment: port overrides (e.g. linuxkm) promise only - * size-based natural alignment. */ + /* aligned allocation roundtrips. */ { static const size_t memalign_aligns[] = { 0U, 1U, 2U, 3U, sizeof(void *), 16U, 64U, 256U @@ -38267,10 +38285,8 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memory_test(void) for (ma_i = 0; ma_i < (word32)(sizeof(memalign_aligns)/sizeof(memalign_aligns[0])); ma_i++) { - size_t sz = (size_t)MEM_TEST_SZ; - if (sz < memalign_aligns[ma_i]) - sz = memalign_aligns[ma_i]; - ret = simple_memalign_test(memalign_aligns[ma_i], sz); + ret = simple_memalign_test(memalign_aligns[ma_i], + (size_t)MEM_TEST_SZ); if (ret != 0) return ret; } @@ -38293,7 +38309,6 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memory_test(void) DYNAMIC_TYPE_TMP_BUFFER) != NULL) { return WC_TEST_RET_ENC_NC; /* non-power-of-2 alignment accepted */ } -#ifndef WC_XMEMALIGN_NATIVE /* The over-allocation implementation must reject a size whose padded * total wraps, without ever reaching the underlying allocator (keeps * sanitizer runs quiet). */ @@ -38301,13 +38316,13 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memory_test(void) DYNAMIC_TYPE_TMP_BUFFER) != NULL) { return WC_TEST_RET_ENC_NC; /* padded-size overflow accepted */ } -#endif #endif /* !WC_XMEMALIGN_PORT_OVERRIDE */ #endif /* WOLFSSL_STATIC_MEMORY || !WOLFSSL_NO_MALLOC */ -#if defined(WOLFSSL_NO_MALLOC) && !defined(WOLFSSL_STATIC_MEMORY) && \ - !defined(WC_XMEMALIGN_PORT_OVERRIDE) - /* stub configuration: aligned allocation must fail cleanly */ +#if defined(NO_WOLFSSL_MEMORY) && defined(WOLFSSL_NO_MALLOC) && \ + !defined(WOLFSSL_STATIC_MEMORY) && !defined(XMALLOC_USER) && \ + !defined(XMALLOC_OVERRIDE) && !defined(WC_XMEMALIGN_PORT_OVERRIDE) + /* XMALLOC() is a NULL stub here: aligned allocation must fail cleanly */ if (XMEMALIGN(16U, 16U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER) != NULL) { return WC_TEST_RET_ENC_NC; } diff --git a/wolfssl/wolfcrypt/memory.h b/wolfssl/wolfcrypt/memory.h index f1ec8cc0333..aa5f9114909 100644 --- a/wolfssl/wolfcrypt/memory.h +++ b/wolfssl/wolfcrypt/memory.h @@ -77,8 +77,10 @@ WOLFSSL_API void* wolfSSL_Malloc(size_t size, const char* func, unsigned int line); WOLFSSL_API void wolfSSL_Free(void *ptr, const char* func, unsigned int line); WOLFSSL_API void* wolfSSL_Realloc(void *ptr, size_t size, const char* func, unsigned int line); + #ifndef WC_XMEMALIGN_PORT_OVERRIDE WOLFSSL_API void* wolfSSL_MemAlign(size_t alignment, size_t size, const char* func, unsigned int line); WOLFSSL_API void wolfSSL_FreeAlign(void *ptr, const char* func, unsigned int line); + #endif #else typedef void *(*wolfSSL_Malloc_cb)(size_t size); typedef void (*wolfSSL_Free_cb)(void *ptr); @@ -87,8 +89,6 @@ WOLFSSL_API void* wolfSSL_Malloc(size_t size); WOLFSSL_API void wolfSSL_Free(void *ptr); WOLFSSL_API void* wolfSSL_Realloc(void *ptr, size_t size); - WOLFSSL_API void* wolfSSL_MemAlign(size_t alignment, size_t size); - WOLFSSL_API void wolfSSL_FreeAlign(void *ptr); #endif /* WOLFSSL_DEBUG_MEMORY */ #endif /* WOLFSSL_STATIC_MEMORY */ @@ -337,16 +337,23 @@ WOLFSSL_LOCAL void wc_MemZero_Check(void* addr, size_t len); #ifndef WOLFSSL_NO_FORCE_ZERO WOLFSSL_API void wc_ForceZero(void *mem, size_t len); +#endif + +/* implementations behind the default XMEMALIGN()/XFREEALIGN() (types.h) */ +#ifndef WC_XMEMALIGN_PORT_OVERRIDE +WOLFSSL_API void *wc_xmemalign(size_t alignment, size_t size, void *heap, + int tag); +WOLFSSL_API void wc_xfreealign(void *ptr, void *heap, int tag); +#endif /* implementations behind XREALLOC_SCRUBBED()/XREALLOCALIGN()/ * XREALLOCALIGN_SCRUBBED() (types.h) -- see contract notes there. - * Guarded with wc_ForceZero(): the scrub arms require ForceZero(). */ + * Unconditional, matching the macros and definitions. */ WOLFSSL_API void *wc_xrealloc_scrubbed(void *ptr, size_t old_size, size_t size, void *heap, int tag); WOLFSSL_API void *wc_xreallocalign(void *ptr, size_t alignment, size_t old_size, size_t size, void *heap, int tag, int scrub); -#endif #ifndef WOLFSSL_NO_CONST_CMP WOLFSSL_API int wc_ConstantCompare(const byte* a, const byte* b, int length); diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index 566fc7a0330..1ac15e95d9d 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -956,12 +956,6 @@ enum { #endif #define XREALLOC(p, n, h, t) ((void)(h), (void)(t), \ wolfSSL_Realloc((p), (n))) - #ifndef WC_XMEMALIGN_PORT_OVERRIDE - #define XMEMALIGN(a, s, h, t) ((void)(h), (void)(t), \ - wolfSSL_MemAlign((a), (s))) - #define XFREEALIGN(p, h, t) do { (void)(h); (void)(t); \ - wolfSSL_FreeAlign((p)); } while (0) - #endif #endif /* WOLFSSL_DEBUG_MEMORY */ #endif /* WOLFSSL_STATIC_MEMORY */ #endif @@ -971,97 +965,16 @@ enum { #define WC_NO_CONSTRUCTORS #endif -#if !defined(WOLFSSL_HAVE_POSIX_MEMALIGN) && !defined(WOLFSSL_NO_POSIX_MEMALIGN) - #if (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) || \ - (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 600)) - #define WOLFSSL_HAVE_POSIX_MEMALIGN - #endif -#endif - -/* Bottommost aligned-allocation primitives. One pair, one name; the - * implementation is selected at compile time: - * - posix_memalign() (native heap) when sensed available and the build - * permits native heap use. WC_XMEMALIGN_NATIVE is defined so upper - * layers (wolfSSL_MemAlign()) know the allocation bypasses - * XMALLOC/wolfSSL_Malloc() and can apply their own accounting. - * - otherwise, over-allocation via XMALLOC()/XFREE() with a word32 offset - * slot immediately below the returned pointer. heap and tag are honored, - * keeping this correct over static pools, user allocators, and exotic - * ports. - * Both branches share one alignment policy: alignments below sizeof(void *) - * are floored to it, larger non-powers-of-2 are rejected (NULL). */ -#if defined(WC_XMEMALIGN_PORT_OVERRIDE) - /* Use definitions supplied by earlier include (e.g. linuxkm_wc_port.h or user_settings.h) */ -#elif defined(WOLFSSL_NO_MALLOC) && !defined(WOLFSSL_STATIC_MEMORY) - static WC_INLINE void *wc_xmemalign(size_t alignment, size_t req_size, void *heap, int tag) { - (void)alignment; - (void)req_size; - (void)heap; - (void)tag; - return NULL; - } - static WC_INLINE void wc_xfreealign(void *ptr, void *heap, int tag) { - (void)ptr; - (void)heap; - (void)tag; - } -#elif !defined(WOLFSSL_NO_MALLOC) && !defined(WOLFSSL_STATIC_MEMORY) && \ - defined(WOLFSSL_HAVE_POSIX_MEMALIGN) - #define WC_XMEMALIGN_NATIVE - static WC_INLINE void *wc_xmemalign(size_t alignment, size_t size, void *heap, int tag) { - void *ret = NULL; - (void)heap; - (void)tag; - /* posix_memalign() EINVALs on alignments that are not power-of-2 - * multiples of sizeof(void *), and sanitizers escalate that to an - * abort. */ - if (alignment < sizeof(void *)) - alignment = sizeof(void *); - else if ((alignment & (alignment - 1U)) != 0) - return NULL; - if (posix_memalign(&ret, alignment, size) != 0) /* native heap */ - return NULL; - else - return ret; - } - #define wc_xfreealign(ptr, h, t) do { (void)(h); (void)(t); free(ptr); } while (0) /* native heap */ -#else - static WC_INLINE void *wc_xmemalign(size_t alignment, size_t req_size, void *heap, int tag) { - void *alloc; - char *ret; - size_t alloc_size; - - /* silently tolerate pathologically small alignments, but reject larger - * unaligned alignments. */ - if (alignment < sizeof(void *)) - alignment = sizeof(void *); - else if ((alignment & (alignment-1U)) != 0) - return NULL; - alloc_size = req_size + sizeof(word32) + alignment - 1U; - if (alloc_size < req_size) - return NULL; - alloc = XMALLOC(alloc_size, heap, tag); - if (alloc == NULL) - return NULL; - ret = (char *)alloc + sizeof(word32) + alignment - 1U; - ret = (char *)((wc_ptr_t)ret & ~((wc_ptr_t)alignment - 1U)); - *(word32 *)(void *)(ret - sizeof(word32)) = (word32)(ret - (char *)alloc); - return (void *)ret; - } - static WC_INLINE void wc_xfreealign(void *ptr, void *heap, int tag) { - (void)tag; - if (ptr != NULL) { - char *p = (char *)ptr; - XFREE(p - *(word32 *)(void *)(p - sizeof(word32)), heap, tag); - } - } -#endif - +/* Bottommost aligned-allocation primitives. By default these are + * wc_xmemalign()/wc_xfreealign() (memory.c), which over-allocate via + * XMALLOC()/XFREE() so heap and tag are honored, and so fail exactly when + * XMALLOC() does (e.g. WOLFSSL_NO_MALLOC without a usable allocator). See + * the notes there. */ #ifndef XMEMALIGN + /* Not a port override (e.g. linuxkm_wc_port.h or user_settings.h) or the + * WOLFSSL_DEBUG_MEMORY wolfSSL_MemAlign() binding. */ #define XMEMALIGN(a, s, h, t) wc_xmemalign(a, s, h, t) -#endif -#ifndef XFREEALIGN - #define XFREEALIGN(p, h, t) wc_xfreealign(p, h, t) + #define XFREEALIGN(p, h, t) do { wc_xfreealign((p), (h), (t)); } while (0) #endif /* Reallocation with scrubbing and/or alignment guarantees, composed over the @@ -1359,7 +1272,9 @@ enum { #define WC_CALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ do { \ WC_ALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP); \ - XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE) * (VAR_SIZE)); \ + if ((VAR_NAME) != NULL) { \ + XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE) * (VAR_SIZE)); \ + } \ } while (0) #define WC_FREE_VAR(VAR_NAME, HEAP) \ @@ -1371,7 +1286,6 @@ enum { #define WC_FREE_VAR_ALIGNED_EX(VAR_NAME, HEAP, TYPE) \ XFREEALIGN(VAR_NAME, (HEAP), TYPE) #define WC_DECLARE_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) \ - wc_static_assert(((VAR_SIZE) % sizeof(VAR_TYPE)) == 0); \ WC_DECLARE_HEAP_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) #define WC_DECLARE_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ wc_static_assert(((VAR_SIZE) % sizeof(VAR_TYPE)) == 0); \ @@ -1394,9 +1308,23 @@ enum { WC_FREE_HEAP_ARRAY_ALIGNED(VAR_NAME, VAR_ITEMS, HEAP) #else #undef WC_DECLARE_VAR_IS_HEAP_ALLOC + /* Stack _ALIGNED variants rely on XALIGNED(), so unlike the heap + * (XMEMALIGN()) variants, alignment is only as good as the compiler's: + * XALIGNED() is a null expansion on compilers without a known alignment + * attribute (e.g. Open Watcom), giving no alignment guarantee there. IAR + * ARM cannot align stack variables beyond 8 bytes (see WOLFSSL_ALIGN()), + * so larger requests are rejected at compile time. */ + #if defined(__ICCARM__) + #define WC_STACK_ALIGN_ASSERT(VAR_ALIGN) \ + wc_static_assert((VAR_ALIGN) <= 8) + #else + #define WC_STACK_ALIGN_ASSERT(VAR_ALIGN) \ + wc_static_assert((VAR_ALIGN) > 0) + #endif #define WC_DECLARE_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) \ VAR_TYPE VAR_NAME[VAR_SIZE] #define WC_DECLARE_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ + WC_STACK_ALIGN_ASSERT(VAR_ALIGN); \ XALIGNED(VAR_ALIGN) VAR_TYPE VAR_NAME[VAR_SIZE] #define WC_ALLOC_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) WC_DO_NOTHING #define WC_ALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) WC_DO_NOTHING @@ -1408,11 +1336,11 @@ enum { #define WC_CALLOC_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) \ XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE)) #define WC_CALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ - XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE)) + XMEMSET(VAR_NAME, 0, sizeof(VAR_NAME)) #define WC_CALLOC_VAR_EX(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP, TY, ONFAIL)\ XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE)) #define WC_CALLOC_VAR_ALIGNED_EX(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP, TY, ONFAIL)\ - XMEMSET(VAR_NAME, 0, sizeof(VAR_TYPE)) + XMEMSET(VAR_NAME, 0, sizeof(VAR_NAME)) #define WC_FREE_VAR(VAR_NAME, HEAP) WC_DO_NOTHING \ /* nothing to free, it's stack */ #define WC_FREE_VAR_ALIGNED(VAR_NAME, HEAP) WC_DO_NOTHING \ @@ -1425,6 +1353,7 @@ enum { #define WC_DECLARE_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ wc_static_assert(((VAR_SIZE) % sizeof(VAR_TYPE)) == 0); \ wc_static_assert(((VAR_SIZE) % (VAR_ALIGN)) == 0); \ + WC_STACK_ALIGN_ASSERT(VAR_ALIGN); \ XALIGNED(VAR_ALIGN) VAR_TYPE VAR_NAME[VAR_ITEMS][(VAR_SIZE) / sizeof(VAR_TYPE)] /* NOLINT(bugprone-sizeof-expression) */ #define WC_ARRAY_ARG(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE) \ VAR_TYPE VAR_NAME[VAR_ITEMS][(VAR_SIZE) / sizeof(VAR_TYPE)] /* NOLINT(bugprone-sizeof-expression) */ From 9e875abf2bf4dd58e5ac7efc818e1109dc30c142 Mon Sep 17 00:00:00 2001 From: Aidan Keefe Date: Fri, 30 Oct 2026 15:14:06 -0600 Subject: [PATCH 3/5] Names are final and docs are in New set of tests and clean up --- doc/dox_comments/header_files/memory.h | 182 +++++++++++++++ doc/dox_comments/header_files/types.h | 145 ++++++++++++ linuxkm/linuxkm_wc_port.h | 14 +- wolfcrypt/src/memory.c | 259 ++++++++++----------- wolfcrypt/test/test.c | 304 ++++++++++++++++++++++--- wolfssl/wolfcrypt/memory.h | 34 +-- wolfssl/wolfcrypt/types.h | 105 ++++----- 7 files changed, 778 insertions(+), 265 deletions(-) diff --git a/doc/dox_comments/header_files/memory.h b/doc/dox_comments/header_files/memory.h index b5430758a30..99881d76b5f 100644 --- a/doc/dox_comments/header_files/memory.h +++ b/doc/dox_comments/header_files/memory.h @@ -636,3 +636,185 @@ void wc_UnloadStaticMemory(WOLFSSL_HEAP_HINT* heap); int wolfSSL_StaticBufferSz_ex(unsigned int listSz, const word32 *sizeList, const word32 *distList, byte* buffer, word32 sz, int flag); + +/*! + \ingroup Memory + + \brief Aligned allocation over XMALLOC(), honoring heap and tag (static + memory pools, wolfSSL_SetAllocators() callbacks, user allocators, ports). + Over-allocates and stores a small header just below the returned pointer. + Called through macro XMALLOCALIGN. + + \return pointer to aligned allocation + \return NULL if alignment is not 0 or a power of 2, the padded size + overflows, or the allocation fails. + + \param alignment 0 or a power of 2; values below 2*sizeof(size_t) are + raised to it + \param size size, in bytes, of the memory to allocate + \param heap heap hint to use for memory. Can be NULL + \param tag dynamic type (see DYNAMIC_TYPE_ list in types.h) + + _Example_ + \code + byte* buf = (byte*)wc_MallocAlign(64, 1024, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + ... + wc_FreeAlign(buf, NULL, DYNAMIC_TYPE_TMP_BUFFER); + \endcode + + \sa wc_FreeAlign + \sa wc_ReallocAlign + \sa XMALLOCALIGN +*/ +void *wc_MallocAlign(size_t alignment, size_t size, void *heap, int tag); + +/*! + \ingroup Memory + + \brief Frees memory returned by wc_MallocAlign(), wc_ReallocAlign() or + wc_ReallocAlignScrubbed(), by calling XFREE(). Aligned and plain + (XMALLOC()) allocations must not be mixed. Not called directly by + wolfSSL, but through macro XFREEALIGN. + + \return void + + \param ptr aligned pointer to free. NULL is a no-op. + \param heap heap hint used for the allocation. Can be NULL + \param tag dynamic type used for the allocation + + _Example_ + \code + byte* buf = (byte*)wc_MallocAlign(64, 1024, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + ... + wc_FreeAlign(buf, NULL, DYNAMIC_TYPE_TMP_BUFFER); + \endcode + + \sa wc_MallocAlign + \sa XFREEALIGN +*/ +void wc_FreeAlign(void *ptr, void *heap, int tag); + +/*! + \ingroup Memory + + \brief Reallocates memory from wc_MallocAlign(), using XMALLOC() and + XFREE(). Never reallocates in place: a new block aligned to alignment is + allocated, the leading min(old size, size) bytes are copied, and ptr is + freed. Not called directly by wolfSSL, but through macro XREALLOCALIGN. + + \return pointer to the new aligned block on success. + \return NULL when size is 0 (ptr is freed), or on failure (ptr is left + intact, including when alignment is rejected). + + \param ptr NULL, or memory from wc_MallocAlign()/wc_ReallocAlign*(). + A NULL ptr behaves as wc_MallocAlign(). + \param alignment as for wc_MallocAlign() + \param size new size in bytes + \param heap heap hint. Can be NULL + \param tag dynamic type + + _Example_ + \code + byte* buf2 = NULL; + byte* buf = (byte*)wc_MallocAlign(64, 1024, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (buf == NULL) + - fail - + + buf2 = (byte*)wc_ReallocAlign(buf, 64, 4000, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (buf2 == NULL) + - fail - + ... + + wc_FreeAlign(buf2, NULL, DYNAMIC_TYPE_TMP_BUFFER); + \endcode + + \sa wc_ReallocAlignScrubbed + \sa wc_MallocAlign + \sa XREALLOCALIGN +*/ +void *wc_ReallocAlign(void *ptr, size_t alignment, size_t size, void *heap, + int tag); + +/*! + \ingroup Memory + + \brief As wc_ReallocAlign(), but the old block is zeroized before it is + freed, including when size is 0. On failure the old block is left intact + and not zeroized. Not called directly by wolfSSL, but through macro + XREALLOCALIGN_SCRUBBED. + + \return pointer to the new aligned block on success. + \return NULL when size is 0 (ptr is zeroized and freed), or on failure. + + \param ptr NULL, or memory from wc_MallocAlign()/wc_ReallocAlign*() + \param alignment as for wc_MallocAlign() + \param size new size in bytes + \param heap heap hint. Can be NULL + \param tag dynamic type + + _Example_ + \code + byte* buf2 = NULL; + byte* buf = (byte*)wc_MallocAlign(64, 1024, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (buf == NULL) + - fail - + + buf2 = (byte*)wc_ReallocAlignScrubbed(buf, 64, 4000, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (buf2 == NULL) + - fail - + + - buf freed and zeroed post ReallocAlignScrubbed- + ... + + wc_FreeAlign(buf2, NULL, DYNAMIC_TYPE_TMP_BUFFER); + \endcode + + \sa wc_ReallocAlign + \sa XREALLOCALIGN_SCRUBBED +*/ +void *wc_ReallocAlignScrubbed(void *ptr, size_t alignment, size_t size, + void *heap, int tag); + +/*! + \ingroup Memory + + \brief Reallocates memory from XMALLOC(), zeroizing the old block before + it is freed. Never reallocates in place: a new block is allocated with + XMALLOC(), the leading min(old_size, size) bytes are copied, old_size + bytes of ptr are zeroized, and ptr is freed with XFREE(). Not called + directly by wolfSSL, but through macro XREALLOC_SCRUBBED. + + \return pointer to the new block on success. + \return NULL when size is 0 (ptr is zeroized and freed), or on failure + (ptr is left intact and not zeroized). + + \param ptr NULL, or memory from XMALLOC()/XREALLOC(). A NULL ptr behaves + as XMALLOC(size). + \param old_size current size of ptr in bytes. Must not exceed the real + allocation size, or the copy and zeroization overrun it. + \param size new size in bytes + \param heap heap hint. Can be NULL + \param tag dynamic type + + _Example_ + \code + byte* key = (byte*)XMALLOC(32, NULL, DYNAMIC_TYPE_TMP_BUFFER); + byte* tmp; + ... + tmp = (byte*)wc_ReallocScrubbed(key, 32, 64, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (tmp != NULL) + key = tmp; + \endcode + + \sa wc_ReallocAlignScrubbed + \sa XREALLOC_SCRUBBED +*/ +void *wc_ReallocScrubbed(void *ptr, size_t old_size, size_t size, + void *heap, int tag); diff --git a/doc/dox_comments/header_files/types.h b/doc/dox_comments/header_files/types.h index b17bb4306b1..457c356e030 100644 --- a/doc/dox_comments/header_files/types.h +++ b/doc/dox_comments/header_files/types.h @@ -142,6 +142,151 @@ void* XREALLOC(void *p, size_t n, void* heap, int type); */ void XFREE(void *p, void* heap, int type); +/*! + \ingroup Memory + + \brief This is not actually a function, but rather a preprocessor macro, + which allocates memory aligned to a given boundary. By default it maps to + wc_MallocAlign(), which over-allocates through XMALLOC() and stores a + small header just below the returned pointer, so heap and type reach + every allocator (static memory, wolfSSL_SetAllocators() callbacks, + XMALLOC_USER, ports). A port or user may supply its own implementation, + but XMALLOCALIGN, XFREEALIGN, XREALLOCALIGN and XREALLOCALIGN_SCRUBBED + must then all be defined together. + Memory from XMALLOCALIGN must be released with XFREEALIGN, never XFREE, + and XFREEALIGN must not be used on memory from XMALLOC. + + \return pointer to n bytes aligned to a on success. + \return NULL if a is not 0 or a power of 2, or the allocation fails. + + \param a alignment: 0 or a power of 2. Values below 2*sizeof(size_t) + (including 0) are raised to it. + \param n size, in bytes, of the memory to allocate + \param h pointer to the heap to use + \param t memory allocation type for user hints. See enum in types.h + + _Example_ + \code + byte* buf = (byte*)XMALLOCALIGN(64, 1024, NULL, DYNAMIC_TYPE_TMP_BUFFER); + if (buf == NULL) { + return MEMORY_E; + } + ... + XFREEALIGN(buf, NULL, DYNAMIC_TYPE_TMP_BUFFER); + \endcode + + \sa XFREEALIGN + \sa XREALLOCALIGN + \sa wc_MallocAlign +*/ +void* XMALLOCALIGN(size_t a, size_t n, void* heap, int type); + +/*! + \ingroup Memory + + \brief This is not actually a function, but rather a preprocessor macro, + which frees memory from XMALLOCALIGN, XREALLOCALIGN or + XREALLOCALIGN_SCRUBBED. By default it maps to wc_FreeAlign(). + + \return none No returns. + + \param p aligned pointer to free. NULL is a no-op. + \param h pointer to the heap used for the allocation + \param t memory allocation type used for the allocation + + \sa XMALLOCALIGN + \sa wc_FreeAlign +*/ +void XFREEALIGN(void *p, void* heap, int type); + +/*! + \ingroup Memory + + \brief This is not actually a function, but rather a preprocessor macro, + which reallocates memory from XMALLOCALIGN. It never reallocates in + place: a new block aligned to a is allocated, the leading + min(old size, n) bytes are copied, and p is freed. By default it maps to + wc_ReallocAlign(). + + \return pointer to the new aligned block on success. + \return NULL when n is 0 (p is freed), or on failure (p is left intact, + including when a is rejected). + + \param p NULL, or memory from XMALLOCALIGN/XREALLOCALIGN*. A NULL p + behaves as XMALLOCALIGN. + \param a alignment, as for XMALLOCALIGN + \param n new size in bytes + \param h pointer to the heap to use + \param t memory allocation type for user hints. See enum in types.h + + _Example_ + \code + byte* buf = (byte*)XMALLOCALIGN(64, 128, NULL, DYNAMIC_TYPE_TMP_BUFFER); + byte* tmp; + ... + tmp = (byte*)XREALLOCALIGN(buf, 64, 256, NULL, DYNAMIC_TYPE_TMP_BUFFER); + if (tmp != NULL) + buf = tmp; + \endcode + + \sa XREALLOCALIGN_SCRUBBED + \sa XMALLOCALIGN + \sa wc_ReallocAlign +*/ +void* XREALLOCALIGN(void *p, size_t a, size_t n, void* heap, int type); + +/*! + \ingroup Memory + + \brief This is not actually a function, but rather a preprocessor macro. + As XREALLOCALIGN, but the old block is zeroized before it is freed, + including when n is 0. On failure the old block is left intact and not + zeroized. Use for buffers holding sensitive data. By default it maps to + wc_ReallocAlignScrubbed(). + + \return pointer to the new aligned block on success. + \return NULL when n is 0 (p is zeroized and freed), or on failure. + + \param p NULL, or memory from XMALLOCALIGN/XREALLOCALIGN* + \param a alignment, as for XMALLOCALIGN + \param n new size in bytes + \param h pointer to the heap to use + \param t memory allocation type for user hints. See enum in types.h + + \sa XREALLOCALIGN + \sa wc_ReallocAlignScrubbed +*/ +void* XREALLOCALIGN_SCRUBBED(void *p, size_t a, size_t n, void* heap, + int type); + +/*! + \ingroup Memory + + \brief This is not actually a function, but rather a preprocessor macro, + which reallocates memory from XMALLOC and zeroizes the old block before + it is freed. It never reallocates in place: a new block is allocated with + XMALLOC, the leading min(old_size, n) bytes are copied, old_size bytes of + p are zeroized, and p is freed with XFREE. By default it maps to + wc_ReallocScrubbed(). + + \return pointer to the new block on success. + \return NULL when n is 0 (p is zeroized and freed), or on failure (p is + left intact and not zeroized). + + \param p NULL, or memory from XMALLOC/XREALLOC. A NULL p behaves as + XMALLOC. + \param old_size current size of p in bytes. Must not exceed the real + allocation size, or the copy and zeroization overrun it. + \param n new size in bytes + \param h pointer to the heap to use + \param t memory allocation type for user hints. See enum in types.h + + \sa XREALLOCALIGN_SCRUBBED + \sa wc_ReallocScrubbed +*/ +void* XREALLOC_SCRUBBED(void *p, size_t old_size, size_t n, void* heap, + int type); + /*! \ingroup Math diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 124979f90b5..8b2abcc2252 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -1,4 +1,4 @@ -/* linuxkm_wc_port.hlunuc +/* linuxkm_wc_port.h * * Copyright (C) 2006-2026 wolfSSL Inc. * @@ -2419,18 +2419,6 @@ #if defined(USE_KVREALLOC) || !defined(USE_KVMALLOC) #define XREALLOC(p, n, h, t) __extension__ ({(void)(h); (void)(t); realloc(p, n);}) #endif -/* Since linux commit 59bb47985c (5.4), kmalloc with power-of-2 size is - * aligned to the size, and malloc() rounds up via WC_LINUXKM_ROUND_UP_P_OF_2, - * so allocating max(a, s) suffices. Older kernels use the wc_xmemalign() - * shim in memory.c. */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) - #define XMEMALIGN(a, s, h, t) __extension__ ({ \ - size_t _wc_a = (a), _wc_s = (s); \ - (void)(h); (void)(t); \ - malloc((_wc_s < _wc_a) ? _wc_a : _wc_s); }) - #define XFREEALIGN(p, h, t) do { (void)(h); (void)(t); free(p); } while (0) -#endif - #endif /* !XMALLOC_USER && !XMALLOC_OVERRIDE */ #endif diff --git a/wolfcrypt/src/memory.c b/wolfcrypt/src/memory.c index c3ce00d4714..e9a500ce67d 100644 --- a/wolfcrypt/src/memory.c +++ b/wolfcrypt/src/memory.c @@ -79,60 +79,102 @@ void *z_realloc(void *ptr, size_t size) #define realloc z_realloc #endif -/* Alignment policy shared by the aligned allocation and reallocation paths: - * alignments below the floor (the larger of sizeof(void *) and the word32 - * offset slot) are raised to it, and larger non-powers-of-2 are rejected. - * Returns the effective alignment, or 0 if rejected. */ -#define WC_ALIGN_FLOOR (sizeof(void *) > sizeof(word32) ? \ - sizeof(void*) : sizeof(word32)) -static WC_INLINE size_t wc_xmemalign_floor(size_t alignment) +/* + * v what is returned + * |padding|offset|size| memory | + * |-----(offset)------|--(size)--| + * */ +typedef size_t wc_align_metadata_member; +typedef struct WC_ALIGN_METADATA { + wc_align_metadata_member offset; + wc_align_metadata_member size; +}WC_ALIGN_METADATA; +/* Floor of 2*sizeof(size_t) (8 on ILP32, 16 on LP64) keeps the metadata + * aligned and, on common ABIs, makes XMALLOCALIGN() at least as aligned as + * XMALLOC(), so small or 0 alignments are a safe drop-in for it. */ +#define WC_ALIGN_FLOOR (sizeof(void *) > sizeof(WC_ALIGN_METADATA) ? \ + sizeof(void*) : sizeof(WC_ALIGN_METADATA)) + +/* Reject non-powers-of-2, then raise alignment to the floor. Returns + * effective alignment, or 0 if rejected. */ +static WC_INLINE size_t wc_MallocAlignFloor(size_t alignment) { - if (alignment < WC_ALIGN_FLOOR) - return WC_ALIGN_FLOOR; if ((alignment & (alignment - 1U)) != 0) return 0; + if (alignment < WC_ALIGN_FLOOR) + return WC_ALIGN_FLOOR; return alignment; } -#ifndef WC_XMEMALIGN_PORT_OVERRIDE -/* Aligned-allocation shim layout, shared by wc_xmemalign()/wc_xfreealign() - * and wolfSSL_MemAlign()/wolfSSL_FreeAlign(): over-allocate, align within - * the block, and record the offset to its start in a word32 slot immediately - * below the returned pointer. +/* Aligned-allocation shim (wc_MallocAlign()): + * over-allocate, align, and store offset and size just below the pointer. * * Size of the underlying allocation for an aligned request, or 0 if the * alignment is rejected or the padded size wraps. *alignment is floored - * in place per wc_xmemalign_floor(). */ -static WC_INLINE size_t wc_xmemalign_alloc_size(size_t *alignment, - size_t req_size) + * in place per wc_MallocAlignFloor(). */ +static WC_INLINE size_t wc_MallocAlignAllocSize(size_t *alignment, + size_t req_size) { size_t alloc_size; - *alignment = wc_xmemalign_floor(*alignment); + *alignment = wc_MallocAlignFloor(*alignment); if (*alignment == 0) return 0; - alloc_size = req_size + sizeof(word32) + *alignment - 1U; + alloc_size = req_size + (sizeof(WC_ALIGN_METADATA)) + *alignment - 1U; if (alloc_size < req_size) return 0; return alloc_size; } -/* Aligned pointer within alloc, with its offset recorded just below. */ -static WC_INLINE void *wc_xmemalign_place(void *alloc, size_t alignment) +/* Aligned pointer within alloc. offset is measured from the returned + * pointer back to alloc. + * v what is returned + * |padding|offset|size| memory | + * |-----(offset)------|--(size)--| + * */ +static WC_INLINE void *wc_MallocAlignPlace(void *alloc, size_t alignment, + size_t req_size) { - char *ret = (char *)alloc + sizeof(word32) + alignment - 1U; + char *ret = (char *)alloc + sizeof(WC_ALIGN_METADATA) + alignment - 1U; + WC_ALIGN_METADATA *metadata; + ret = (char *)((wc_ptr_t)ret & ~((wc_ptr_t)alignment - 1U)); - *(word32 *)(void *)(ret - sizeof(word32)) = (word32)(ret - (char *)alloc); + metadata = (WC_ALIGN_METADATA *)(void *)(ret - sizeof(WC_ALIGN_METADATA)); + metadata->offset = (wc_align_metadata_member)(ret - (char *)alloc); + metadata->size = (wc_align_metadata_member)req_size; return (void *)ret; } +static WC_INLINE wc_align_metadata_member +wc_MallocAlignSize(void* ptr) +{ + char *p = (char *)ptr; + p -= sizeof(WC_ALIGN_METADATA); + return ((WC_ALIGN_METADATA*)(void*)p)->size; +} + /* Underlying allocation for an aligned pointer (non-NULL). */ -static WC_INLINE void *wc_xmemalign_base(void *ptr) +static WC_INLINE void *wc_MallocAlignBase(void *ptr) { char *p = (char *)ptr; - return (void *)(p - *(word32 *)(void *)(p - sizeof(word32))); + /* offset slot holds the distance from ptr back to the allocation */ + WC_ALIGN_METADATA* metaData = + (WC_ALIGN_METADATA*)(void *)(p - (sizeof(WC_ALIGN_METADATA))); + return (void *)(p - metaData->offset); +} + +static void wc_ReallocScrub(void *ptr, size_t len) +{ +#ifdef WOLFSSL_NO_FORCE_ZERO + /* volatile stores: a plain XMEMSET() just ahead of XFREE() is a dead + * store the compiler may remove. */ + volatile byte *p = (volatile byte *)ptr; + while (len-- > 0) + *p++ = 0; +#else + ForceZero(ptr, len); +#endif } -#endif /* !WC_XMEMALIGN_PORT_OVERRIDE */ #ifdef USE_WOLFSSL_MEMORY @@ -541,44 +583,6 @@ void wolfSSL_Free(void *ptr) } } -#if !defined(WC_XMEMALIGN_PORT_OVERRIDE) && defined(WOLFSSL_DEBUG_MEMORY) -/* Aligned allocation wrappers using the wc_xmemalign() shim layout. The - * underlying memory comes from wolfSSL_Malloc()/wolfSSL_Free(), called here - * directly so that the caller's func/line reach allocator callbacks (e.g. - * WOLFSSL_TRACK_MEMORY) instead of this file's. Non-debug builds use - * wc_xmemalign()/wc_xfreealign(), which reach wolfSSL_Malloc()/wolfSSL_Free() - * through XMALLOC()/XFREE() with identical effect. */ -void* wolfSSL_MemAlign(size_t alignment, size_t size, const char* func, - unsigned int line) -{ - void* res = NULL; - void* alloc; - size_t alloc_size = wc_xmemalign_alloc_size(&alignment, size); - - if (alloc_size != 0) { - alloc = wolfSSL_Malloc(alloc_size, func, line); - if (alloc != NULL) - res = wc_xmemalign_place(alloc, alignment); - } - -#if defined(WOLFSSL_DEBUG_MEMORY_PRINT) && !defined(WOLFSSL_TRACK_MEMORY) - fprintf(stderr, "MemAlign: %p -> %u (align %u) at %s:%u\n", res, - (word32)size, (word32)alignment, func, line); -#endif - - return res; -} - -void wolfSSL_FreeAlign(void *ptr, const char* func, unsigned int line) -{ -#if defined(WOLFSSL_DEBUG_MEMORY_PRINT) && !defined(WOLFSSL_TRACK_MEMORY) - fprintf(stderr, "FreeAlign: %p at %s:%u\n", ptr, func, line); -#endif - if (ptr != NULL) - wolfSSL_Free(wc_xmemalign_base(ptr), func, line); -} -#endif /* !WC_XMEMALIGN_PORT_OVERRIDE && WOLFSSL_DEBUG_MEMORY */ - #ifdef WOLFSSL_DEBUG_MEMORY void* wolfSSL_Realloc(void *ptr, size_t size, const char* func, unsigned int line) #else @@ -1830,15 +1834,14 @@ void wc_ForceZero(void *mem, size_t len) } #endif -#ifndef WC_XMEMALIGN_PORT_OVERRIDE -/* Default XMEMALIGN()/XFREEALIGN(): the shim layout over XMALLOC()/XFREE(), +/* Default XMALLOCALIGN()/XFREEALIGN(): the shim layout over XMALLOC()/XFREE(), * so heap and tag are honored, keeping this correct over static pools, user * allocators, allocator callbacks, and exotic ports. Alignment policy per - * wc_xmemalign_floor(). */ -void *wc_xmemalign(size_t alignment, size_t size, void *heap, int tag) + * wc_MallocAlignFloor(). */ +void *wc_MallocAlign(size_t alignment, size_t size, void *heap, int tag) { void *alloc; - size_t alloc_size = wc_xmemalign_alloc_size(&alignment, size); + size_t alloc_size = wc_MallocAlignAllocSize(&alignment, size); (void)heap; (void)tag; @@ -1848,41 +1851,27 @@ void *wc_xmemalign(size_t alignment, size_t size, void *heap, int tag) alloc = XMALLOC(alloc_size, heap, tag); if (alloc == NULL) return NULL; - return wc_xmemalign_place(alloc, alignment); + return wc_MallocAlignPlace(alloc, alignment, size); } -void wc_xfreealign(void *ptr, void *heap, int tag) +void wc_FreeAlign(void *ptr, void *heap, int tag) { (void)heap; (void)tag; if (ptr != NULL) - XFREE(wc_xmemalign_base(ptr), heap, tag); -} -#endif - -static void wc_xrealloc_scrub(void *ptr, size_t len) -{ -#ifdef WOLFSSL_NO_FORCE_ZERO - /* volatile stores: a plain XMEMSET() just ahead of XFREE() is a dead - * store the compiler may remove. */ - volatile byte *p = (volatile byte *)ptr; - while (len-- > 0) - *p++ = 0; -#else - ForceZero(ptr, len); -#endif + XFREE(wc_MallocAlignBase(ptr), heap, tag); } /* Reallocation with scrubbing and/or alignment guarantees: implementations * behind the XREALLOC_SCRUBBED()/XREALLOCALIGN()/XREALLOCALIGN_SCRUBBED() - * macros (types.h), composed over the XREALLOC/XMALLOC/XFREE/XMEMALIGN/ - * XFREEALIGN macro bindings so that port overrides, wolfSSL memory - * callbacks, and static pools are all honored. See the contract notes at - * the macro definitions. On failure (NULL return) the original allocation - * is always intact and, for scrub, unscrubbed. */ -void *wc_xrealloc_scrubbed(void *ptr, size_t old_size, size_t size, - void *heap, int tag) + * macros (types.h), composed over the XMALLOC/XFREE + * macro bindings so that port overrides, wolfSSL memory + * callbacks, and static pools are all honored. See the contract notes above + * the XMALLOCALIGN() definitions in types.h. On failure (NULL return) the + * original allocation is always intact and, for scrub, unscrubbed. */ +void *wc_ReallocScrubbed(void *ptr, size_t old_size, size_t size, + void *heap, int tag) { void *new_p; @@ -1892,11 +1881,8 @@ void *wc_xrealloc_scrubbed(void *ptr, size_t old_size, size_t size, if (ptr == NULL && size != 0) return XMALLOC(size, heap, tag); else if (ptr != NULL && size == 0) { - /* realloc-to-zero idiom: scrub, free, and return NULL. Handled - * explicitly (and ahead of all other arms) so the semantics are - * uniform across builds rather than the underlying allocator's - * implementation-defined choice. */ - wc_xrealloc_scrub(ptr, old_size); + /* realloc-to-zero idiom: scrub, free, and return NULL. */ + wc_ReallocScrub(ptr, old_size); XFREE(ptr, heap, tag); return NULL; } @@ -1907,62 +1893,49 @@ void *wc_xrealloc_scrubbed(void *ptr, size_t old_size, size_t size, if (new_p == NULL) return NULL; /* original intact and unscrubbed */ XMEMCPY(new_p, ptr, (old_size < size) ? old_size : size); - wc_xrealloc_scrub(ptr, old_size); + wc_ReallocScrub(ptr, old_size); XFREE(ptr, heap, tag); return new_p; } -void *wc_xreallocalign(void *ptr, size_t alignment, size_t old_size, - size_t size, void *heap, int tag, int scrub) +static void *wc_ReallocAlignCommon(void *ptr, size_t alignment, size_t size, + void *heap, int tag, int scrub) { - void *new_p; + void *new_p = NULL; + wc_align_metadata_member curSize = 0; - (void)heap; - if (size == 0) { - /* realloc-to-zero idiom: free (scrubbed if requested) and return - * NULL. Handled explicitly, ahead of the retention arm, which would - * wrongly keep a small allocation alive. */ - if (scrub && (ptr != NULL)) - wc_xrealloc_scrub(ptr, old_size); - XFREEALIGN(ptr, heap, tag); - return NULL; - } + if (ptr != NULL) + curSize = wc_MallocAlignSize(ptr); - (void)tag; + if (size != 0) { + /* wc_MallocAlign() rejects bad alignments */ + new_p = wc_MallocAlign(alignment, size, heap, tag); + if (new_p == NULL || ptr == NULL) + return new_p; /* on failure, original intact and unscrubbed */ + XMEMCPY(new_p, ptr, (curSize > size) ? size : curSize); + } + /* else realloc-to-zero idiom: scrub, free, and return NULL. */ - if (ptr == NULL) - return XMEMALIGN(alignment, size, heap, tag); - - /* Match the alignment policy of the allocation-side primitives, so the - * shrink-retention test below reasons about the alignment actually in - * effect. */ - alignment = wc_xmemalign_floor(alignment); - if (alignment == 0) - return NULL; /* original intact */ - - /* Shrink retention: when the reclaimable slack cannot exceed the aligned - * allocator's own overhead, compaction is footprint-neutral at best -- - * retain the allocation. Larger shrinks fall through and genuinely - * release memory. The caller forgets the tail [size, old_size), and - * later scrubs cover only size bytes, so scrub the tail now. */ - if ((size <= old_size) && - ((old_size - size) <= ((alignment - 1U) + sizeof(word32))) && - (((wc_ptr_t)ptr & ((wc_ptr_t)alignment - 1U)) == 0)) { - if (scrub && (old_size > size)) - wc_xrealloc_scrub((byte *)ptr + size, old_size - size); - return ptr; - } - - new_p = XMEMALIGN(alignment, size, heap, tag); - if (new_p == NULL) - return NULL; /* original intact and unscrubbed */ - XMEMCPY(new_p, ptr, (old_size < size) ? old_size : size); - if (scrub) - wc_xrealloc_scrub(ptr, old_size); - XFREEALIGN(ptr, heap, tag); + if (ptr != NULL) { + if (scrub) + wc_ReallocScrub(ptr, curSize); + wc_FreeAlign(ptr, heap, tag); + } return new_p; } +void *wc_ReallocAlignScrubbed(void *ptr, size_t alignment, size_t size, + void *heap, int tag) +{ + return wc_ReallocAlignCommon(ptr, alignment, size, heap, tag, 1); +} + +void *wc_ReallocAlign(void *ptr, size_t alignment, size_t size, void *heap, + int tag) +{ + return wc_ReallocAlignCommon(ptr, alignment, size, heap, tag, 0); +} + #ifndef WOLFSSL_NO_CONST_CMP /* Exported version of ConstantCompare(). */ int wc_ConstantCompare(const byte* a, const byte* b, int length) diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 0ae023f6b26..806a683df5e 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -26603,8 +26603,8 @@ static wc_test_ret_t aeskeywrap_caller_aes_test(const keywrapVector* v, wc_test_ret_t ret = 0; int wrapSz = 0, plainSz; - aes = (Aes*)XMEMALIGN(AESKEYWRAP_AES_ALIGN, sizeof(Aes), HEAP_HINT, - DYNAMIC_TYPE_TMP_BUFFER); + aes = (Aes*)XMALLOCALIGN(AESKEYWRAP_AES_ALIGN, sizeof(Aes), HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); if (aes == NULL) return WC_TEST_RET_ENC_EC(MEMORY_E); @@ -37876,7 +37876,7 @@ static wc_test_ret_t simple_memalign_test(size_t alignment, size_t sz) byte* b; int i; - b = (byte*)XMEMALIGN(alignment, sz, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + b = (byte*)XMALLOCALIGN(alignment, sz, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (b == NULL) { return WC_TEST_RET_ENC_I((word32)alignment); } @@ -37932,7 +37932,7 @@ static wc_test_ret_t memalign_macros_test(void) /* XALIGNED() expands to nothing on compilers with no known alignment * attribute, so only assert alignment where the declaration (stack - * build) or XMEMALIGN() (heap build) actually guarantees it. */ + * build) or XMALLOCALIGN() (heap build) actually guarantees it. */ #if defined(WC_DECLARE_VAR_IS_HEAP_ALLOC) || \ (!defined(__WATCOMC__) && \ (defined(__GNUC__) || defined(__llvm__) || \ @@ -37997,7 +37997,7 @@ static wc_test_ret_t memalign_realloc_test(void) int i; /* NULL in: degrades to fresh aligned allocation. */ - p = (byte*)XREALLOCALIGN(NULL, 64U, 0U, 300U, HEAP_HINT, + p = (byte*)XREALLOCALIGN(NULL, 64U, 300U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (p == NULL) return WC_TEST_RET_ENC_EC(MEMORY_E); @@ -38009,7 +38009,7 @@ static wc_test_ret_t memalign_realloc_test(void) p[i] = (byte)i; /* scrubbed grow: contents and alignment preserved. */ - q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 300U, 1600U, HEAP_HINT, + q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 1600U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (q == NULL) { /* original intact on failure, per contract. */ @@ -38028,9 +38028,12 @@ static wc_test_ret_t memalign_realloc_test(void) /* non-power-of-2 alignment: rejected, original intact. */ if (ret == 0) { - if (XREALLOCALIGN(p, 24U, 1580U, 1600U, HEAP_HINT, - DYNAMIC_TYPE_TMP_BUFFER) != NULL) + q = (byte*)XREALLOCALIGN(p, 24U, 1600U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q != NULL) { + p = q; /* old block was released; keep the live one */ ret = WC_TEST_RET_ENC_NC; + } for (i = 0; (ret == 0) && (i < 300); i++) { if (p[i] != (byte)i) ret = WC_TEST_RET_ENC_I((word32)i); @@ -38040,7 +38043,25 @@ static wc_test_ret_t memalign_realloc_test(void) /* large shrink: compaction (may move); contents and alignment * preserved. */ if (ret == 0) { - q = (byte*)XREALLOCALIGN(p, 64U, 1580U, 128U, HEAP_HINT, + q = (byte*)XREALLOCALIGN(p, 64U, 128U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q == NULL) + ret = WC_TEST_RET_ENC_EC(MEMORY_E); /* p intact on failure */ + else { + p = q; + if (((wc_ptr_t)p & 63U) != 0) + ret = WC_TEST_RET_ENC_NC; + for (i = 0; (ret == 0) && (i < 128); i++) { + if (p[i] != (byte)i) + ret = WC_TEST_RET_ENC_I((word32)i); + } + } + } + + /* unscrubbed grow well past any allocator slack: contents and alignment + * preserved, and only the old size is copied (no over-read). */ + if (ret == 0) { + q = (byte*)XREALLOCALIGN(p, 64U, 4096U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (q == NULL) ret = WC_TEST_RET_ENC_EC(MEMORY_E); /* p intact on failure */ @@ -38052,6 +38073,9 @@ static wc_test_ret_t memalign_realloc_test(void) if (p[i] != (byte)i) ret = WC_TEST_RET_ENC_I((word32)i); } + /* utilize the grown extent. */ + if (ret == 0) + XMEMSET(p + 128, 0xA5, 4096U - 128U); } } XFREEALIGN(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); @@ -38092,23 +38116,239 @@ static wc_test_ret_t memalign_realloc_test(void) } XFREE(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); - /* realloc-to-zero idiom: frees and returns NULL. */ + /* realloc-to-zero idiom: frees and returns NULL. A non-NULL return is + * a live allocation, so release it rather than leak. */ if (ret == 0) { - p = (byte*)XMEMALIGN(64U, 128U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + p = (byte*)XMALLOCALIGN(64U, 128U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (p == NULL) return WC_TEST_RET_ENC_EC(MEMORY_E); - if (XREALLOCALIGN_SCRUBBED(p, 64U, 128U, 0U, HEAP_HINT, - DYNAMIC_TYPE_TMP_BUFFER) != NULL) + q = (byte*)XREALLOCALIGN(p, 64U, 0U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q != NULL) { + XFREEALIGN(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); return WC_TEST_RET_ENC_NC; + } + } + if (ret == 0) { + p = (byte*)XMALLOCALIGN(64U, 128U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 0U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q != NULL) { + XFREEALIGN(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_NC; + } + } + if (ret == 0) { p = (byte*)XMALLOC(96U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (p == NULL) return WC_TEST_RET_ENC_EC(MEMORY_E); - if (XREALLOC_SCRUBBED(p, 96U, 0U, HEAP_HINT, - DYNAMIC_TYPE_TMP_BUFFER) != NULL) + q = (byte*)XREALLOC_SCRUBBED(p, 96U, 0U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q != NULL) { + XFREE(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); return WC_TEST_RET_ENC_NC; + } } return ret; } + +/* A port may bind the XMALLOCALIGN() family elsewhere, leaving the generic + * wc_*Align() shim otherwise untested, so call it directly. Bogus-alignment + * rejection and overflow resilience are shim semantics, not part of the + * XMALLOCALIGN() contract. */ +static wc_test_ret_t memalign_shim_test(void) +{ + wc_test_ret_t ret = 0; + byte* p; + byte* q; + int i; + + /* non-power-of-2 alignment: rejected. */ + p = (byte*)wc_MallocAlign(24U, (size_t)MEM_TEST_SZ, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (p != NULL) { + wc_FreeAlign(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_NC; + } + /* padded size that wraps: rejected without reaching the underlying + * allocator (keeps sanitizer runs quiet). */ + p = (byte*)wc_MallocAlign(64U, (size_t)0 - 32U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (p != NULL) { + wc_FreeAlign(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_NC; + } + + /* allocation: aligned and fully usable. */ + p = (byte*)wc_MallocAlign(64U, 300U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + if (((wc_ptr_t)p & 63U) != 0) + ret = WC_TEST_RET_ENC_NC; + for (i = 0; i < 300; i++) + p[i] = (byte)i; + + /* grow: contents and alignment preserved. */ + if (ret == 0) { + q = (byte*)wc_ReallocAlign(p, 64U, 1600U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q == NULL) + ret = WC_TEST_RET_ENC_EC(MEMORY_E); /* p intact on failure */ + else { + p = q; + if (((wc_ptr_t)p & 63U) != 0) + ret = WC_TEST_RET_ENC_NC; + for (i = 0; (ret == 0) && (i < 300); i++) { + if (p[i] != (byte)i) + ret = WC_TEST_RET_ENC_I((word32)i); + } + if (ret == 0) + XMEMSET(p + 300, 0xA5, 1600U - 300U); + } + } + + /* non-power-of-2 alignment: rejected, original intact. */ + if (ret == 0) { + q = (byte*)wc_ReallocAlign(p, 24U, 1600U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q != NULL) { + p = q; /* old block was released; keep the live one */ + ret = WC_TEST_RET_ENC_NC; + } + for (i = 0; (ret == 0) && (i < 300); i++) { + if (p[i] != (byte)i) + ret = WC_TEST_RET_ENC_I((word32)i); + } + } + + /* scrubbed shrink: leading contents and alignment preserved. */ + if (ret == 0) { + q = (byte*)wc_ReallocAlignScrubbed(p, 64U, 128U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q == NULL) + ret = WC_TEST_RET_ENC_EC(MEMORY_E); /* p intact on failure */ + else { + p = q; + if (((wc_ptr_t)p & 63U) != 0) + ret = WC_TEST_RET_ENC_NC; + for (i = 0; (ret == 0) && (i < 128); i++) { + if (p[i] != (byte)i) + ret = WC_TEST_RET_ENC_I((word32)i); + } + } + } + + /* scrubbed realloc-to-zero: releases p and returns NULL. */ + if (ret == 0) { + q = (byte*)wc_ReallocAlignScrubbed(p, 64U, 0U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + p = NULL; + if (q != NULL) { + p = q; + ret = WC_TEST_RET_ENC_NC; + } + } + wc_FreeAlign(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (ret != 0) + return ret; + + /* NULL in: degrades to fresh aligned allocation; then unscrubbed + * realloc-to-zero. */ + p = (byte*)wc_ReallocAlign(NULL, 128U, 64U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + if (((wc_ptr_t)p & 127U) != 0) { + wc_FreeAlign(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_NC; + } + q = (byte*)wc_ReallocAlign(p, 128U, 0U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q != NULL) { + wc_FreeAlign(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_NC; + } + + /* NULL tolerance */ + wc_FreeAlign(NULL, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return 0; +} + +#if defined(WOLFSSL_CHECK_MEM_ZERO) && defined(USE_WOLFSSL_MEMORY) && \ + !defined(WOLFSSL_STATIC_MEMORY) && !defined(WOLFSSL_NO_MALLOC) && \ + !defined(WOLFSSL_PRIVATE_TEST_VIS) +#define MEMALIGN_SCRUB_PATTERN 0xA5 +/* The _SCRUBBED reallocs must zero the old block before releasing it, but + * the old block is gone once they return. Register it with wc_MemZero_Add() + * instead: wolfSSL_Free() checks registered memory is zero at the moment it + * is freed, and aborts if not. On failure the original is intact and still + * registered, so it is zeroed by hand before being freed. */ +static wc_test_ret_t memalign_scrub_test(void) +{ + byte* p; + byte* q; + + /* aligned grow */ + p = (byte*)XMALLOCALIGN(64U, 300U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + XMEMSET(p, MEMALIGN_SCRUB_PATTERN, 300U); + wc_MemZero_Add("memalign_scrub_test aligned grow", p, 300U); + q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 1600U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q == NULL) { + XMEMSET(p, 0, 300U); + XFREEALIGN(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_EC(MEMORY_E); + } + XFREEALIGN(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + + /* aligned realloc-to-zero */ + p = (byte*)XMALLOCALIGN(64U, 300U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + XMEMSET(p, MEMALIGN_SCRUB_PATTERN, 300U); + wc_MemZero_Add("memalign_scrub_test aligned zero", p, 300U); + q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 0U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q != NULL) { + XFREEALIGN(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_NC; + } + + /* unaligned grow */ + p = (byte*)XMALLOC(200U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + XMEMSET(p, MEMALIGN_SCRUB_PATTERN, 200U); + wc_MemZero_Add("memalign_scrub_test unaligned grow", p, 200U); + q = (byte*)XREALLOC_SCRUBBED(p, 200U, 400U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q == NULL) { + XMEMSET(p, 0, 200U); + XFREE(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_EC(MEMORY_E); + } + XFREE(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + + /* unaligned realloc-to-zero */ + p = (byte*)XMALLOC(200U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + if (p == NULL) + return WC_TEST_RET_ENC_EC(MEMORY_E); + XMEMSET(p, MEMALIGN_SCRUB_PATTERN, 200U); + wc_MemZero_Add("memalign_scrub_test unaligned zero", p, 200U); + q = (byte*)XREALLOC_SCRUBBED(p, 200U, 0U, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER); + if (q != NULL) { + XFREE(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + return WC_TEST_RET_ENC_NC; + } + + return 0; +} +#endif /* WOLFSSL_CHECK_MEM_ZERO && USE_WOLFSSL_MEMORY && ... */ #endif /* If successful, returns the first letter of the byte array `in`. @@ -38279,7 +38519,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memory_test(void) /* aligned allocation roundtrips. */ { static const size_t memalign_aligns[] = { - 0U, 1U, 2U, 3U, sizeof(void *), 16U, 64U, 256U + 0U, 2U, 4U, sizeof(void *), 16U, 64U, 256U }; word32 ma_i; for (ma_i = 0; @@ -38302,28 +38542,26 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t memory_test(void) if (ret != 0) return ret; -#ifndef WC_XMEMALIGN_PORT_OVERRIDE - /* Bogus-alignment rejection and overflow resilience are generic- - * implementation semantics, not part of the port-override contract. */ - if (XMEMALIGN(24U, (size_t)MEM_TEST_SZ, HEAP_HINT, - DYNAMIC_TYPE_TMP_BUFFER) != NULL) { - return WC_TEST_RET_ENC_NC; /* non-power-of-2 alignment accepted */ - } - /* The over-allocation implementation must reject a size whose padded - * total wraps, without ever reaching the underlying allocator (keeps - * sanitizer runs quiet). */ - if (XMEMALIGN(64U, (size_t)0 - 32U, HEAP_HINT, - DYNAMIC_TYPE_TMP_BUFFER) != NULL) { - return WC_TEST_RET_ENC_NC; /* padded-size overflow accepted */ - } -#endif /* !WC_XMEMALIGN_PORT_OVERRIDE */ +#if defined(WOLFSSL_CHECK_MEM_ZERO) && defined(USE_WOLFSSL_MEMORY) && \ + !defined(WOLFSSL_STATIC_MEMORY) && !defined(WOLFSSL_NO_MALLOC) && \ + !defined(WOLFSSL_PRIVATE_TEST_VIS) + /* _SCRUBBED variants zero the old block before freeing it */ + ret = memalign_scrub_test(); + if (ret != 0) + return ret; +#endif + + /* generic wc_*Align() shim, whatever XMALLOCALIGN() is bound to */ + ret = memalign_shim_test(); + if (ret != 0) + return ret; #endif /* WOLFSSL_STATIC_MEMORY || !WOLFSSL_NO_MALLOC */ #if defined(NO_WOLFSSL_MEMORY) && defined(WOLFSSL_NO_MALLOC) && \ !defined(WOLFSSL_STATIC_MEMORY) && !defined(XMALLOC_USER) && \ - !defined(XMALLOC_OVERRIDE) && !defined(WC_XMEMALIGN_PORT_OVERRIDE) + !defined(XMALLOC_OVERRIDE) /* XMALLOC() is a NULL stub here: aligned allocation must fail cleanly */ - if (XMEMALIGN(16U, 16U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER) != NULL) { + if (wc_MallocAlign(16U, 16U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER) != NULL) { return WC_TEST_RET_ENC_NC; } #endif diff --git a/wolfssl/wolfcrypt/memory.h b/wolfssl/wolfcrypt/memory.h index aa5f9114909..14807ae1ebc 100644 --- a/wolfssl/wolfcrypt/memory.h +++ b/wolfssl/wolfcrypt/memory.h @@ -77,10 +77,6 @@ WOLFSSL_API void* wolfSSL_Malloc(size_t size, const char* func, unsigned int line); WOLFSSL_API void wolfSSL_Free(void *ptr, const char* func, unsigned int line); WOLFSSL_API void* wolfSSL_Realloc(void *ptr, size_t size, const char* func, unsigned int line); - #ifndef WC_XMEMALIGN_PORT_OVERRIDE - WOLFSSL_API void* wolfSSL_MemAlign(size_t alignment, size_t size, const char* func, unsigned int line); - WOLFSSL_API void wolfSSL_FreeAlign(void *ptr, const char* func, unsigned int line); - #endif #else typedef void *(*wolfSSL_Malloc_cb)(size_t size); typedef void (*wolfSSL_Free_cb)(void *ptr); @@ -330,7 +326,8 @@ WOLFSSL_LOCAL void wc_MemFailCount_Free(void); #ifdef WOLFSSL_CHECK_MEM_ZERO WOLFSSL_LOCAL void wc_MemZero_Init(void); WOLFSSL_LOCAL void wc_MemZero_Free(void); -WOLFSSL_LOCAL void wc_MemZero_Add(const char* name, const void* addr, +/* WOLFSSL_TEST_VIS: test.c registers buffers to check scrubbing. */ +WOLFSSL_TEST_VIS void wc_MemZero_Add(const char* name, const void* addr, size_t len); WOLFSSL_LOCAL void wc_MemZero_Check(void* addr, size_t len); #endif @@ -339,21 +336,24 @@ WOLFSSL_LOCAL void wc_MemZero_Check(void* addr, size_t len); WOLFSSL_API void wc_ForceZero(void *mem, size_t len); #endif -/* implementations behind the default XMEMALIGN()/XFREEALIGN() (types.h) */ -#ifndef WC_XMEMALIGN_PORT_OVERRIDE -WOLFSSL_API void *wc_xmemalign(size_t alignment, size_t size, void *heap, - int tag); -WOLFSSL_API void wc_xfreealign(void *ptr, void *heap, int tag); -#endif +/* implementations behind the default XMALLOCALIGN()/XFREEALIGN() (types.h) */ +WOLFSSL_API void *wc_MallocAlign(size_t alignment, size_t size, void *heap, + int tag); +WOLFSSL_API void wc_FreeAlign(void *ptr, void *heap, int tag); /* implementations behind XREALLOC_SCRUBBED()/XREALLOCALIGN()/ - * XREALLOCALIGN_SCRUBBED() (types.h) -- see contract notes there. + * XREALLOCALIGN_SCRUBBED() (types.h). + * + * only use aligned allocation functions on allocation created by aligned + * allocation functions + * * Unconditional, matching the macros and definitions. */ -WOLFSSL_API void *wc_xrealloc_scrubbed(void *ptr, size_t old_size, - size_t size, void *heap, int tag); -WOLFSSL_API void *wc_xreallocalign(void *ptr, size_t alignment, - size_t old_size, size_t size, void *heap, - int tag, int scrub); +WOLFSSL_API void *wc_ReallocScrubbed(void *ptr, size_t old_size, + size_t size, void *heap, int tag); +WOLFSSL_API void *wc_ReallocAlignScrubbed(void *ptr, size_t alignment, + size_t size, void *heap, int tag); +WOLFSSL_API void *wc_ReallocAlign(void *ptr, size_t alignment, size_t size, + void *heap, int tag); #ifndef WOLFSSL_NO_CONST_CMP WOLFSSL_API int wc_ConstantCompare(const byte* a, const byte* b, int length); diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index 1ac15e95d9d..31476a299c5 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -756,13 +756,16 @@ enum { ((out) = (in1) - (in2), \ /* coverity[INTEGER_OVERFLOW] */ 1)))) -#if defined(XMEMALIGN) != defined(XFREEALIGN) - #error XMEMALIGN and XFREEALIGN must be defined as a matched pair. -#endif -#ifdef XMEMALIGN - /* A port (e.g. linuxkm) supplied the aligned-allocation wrappers before - * this platform chain; suppress the generic implementations below. */ - #define WC_XMEMALIGN_PORT_OVERRIDE +/* Aligned-allocation overrides must be all-or-none: the default + * XREALLOCALIGN*() read the wc_MallocAlign() hidden header, which a port + * XMALLOCALIGN() doesn't write. */ +#if defined(XMALLOCALIGN) || defined(XFREEALIGN) || \ + defined(XREALLOCALIGN) || defined(XREALLOCALIGN_SCRUBBED) + #if !defined(XMALLOCALIGN) || !defined(XFREEALIGN) || \ + !defined(XREALLOCALIGN) || !defined(XREALLOCALIGN_SCRUBBED) + #error XMALLOCALIGN, XFREEALIGN, XREALLOCALIGN and \ + XREALLOCALIGN_SCRUBBED must be overridden together or not at all. + #endif #endif #if defined(HAVE_IO_POOL) @@ -938,12 +941,6 @@ enum { #endif #define XREALLOC(p, n, h, t) ((void)(h), (void)(t), \ wolfSSL_Realloc((p), (n), __func__, __LINE__)) - #ifndef WC_XMEMALIGN_PORT_OVERRIDE - #define XMEMALIGN(a, s, h, t) ((void)(h), (void)(t), \ - wolfSSL_MemAlign((a), (s), __func__, __LINE__)) - #define XFREEALIGN(p, h, t) do { (void)(h); (void)(t); \ - wolfSSL_FreeAlign((p), __func__, __LINE__); } while (0) - #endif #else #define XMALLOC(s, h, t) ((void)(h), (void)(t), \ wolfSSL_Malloc((s))) @@ -965,45 +962,26 @@ enum { #define WC_NO_CONSTRUCTORS #endif -/* Bottommost aligned-allocation primitives. By default these are - * wc_xmemalign()/wc_xfreealign() (memory.c), which over-allocate via - * XMALLOC()/XFREE() so heap and tag are honored, and so fail exactly when - * XMALLOC() does (e.g. WOLFSSL_NO_MALLOC without a usable allocator). See - * the notes there. */ -#ifndef XMEMALIGN - /* Not a port override (e.g. linuxkm_wc_port.h or user_settings.h) or the - * WOLFSSL_DEBUG_MEMORY wolfSSL_MemAlign() binding. */ - #define XMEMALIGN(a, s, h, t) wc_xmemalign(a, s, h, t) - #define XFREEALIGN(p, h, t) do { wc_xfreealign((p), (h), (t)); } while (0) -#endif - -/* Reallocation with scrubbing and/or alignment guarantees, composed over the - * XREALLOC/XMALLOC/XFREE/XMEMALIGN/XFREEALIGN macro bindings so that port - * overrides, wolfSSL memory callbacks, and static pools are all honored. - * - * NOTA BENE: unlike XREALLOC(), these take the current allocated size as an - * explicit argument, ordered chronologically: old_size then new size. The - * _SCRUBBED variants ForceZero() the old allocation before releasing it, and - * take responsibility for the copy away from the underlying allocator, so no - * secret bytes are ever abandoned on a free list. On failure (NULL return) the - * original allocation is always intact and, for _SCRUBBED, unscrubbed -- - * matching the realloc() contract. Passing a size of 0 frees the allocation - * (scrubbed for _SCRUBBED) and returns NULL, matching the classic - * realloc-to-zero idiom. Never pass an XMEMALIGN() allocation to - * XREALLOC()/XREALLOC_SCRUBBED()/XFREE(), nor a plain allocation to the ALIGN - * variants: regular or ALIGN variants must be used consistently across all heap - * operations for a given allocation, due to the fallback strategy. */ +/* Aligned allocation via over-allocation and manual alignment, with a hidden + * header below the returned pointer, layered on XMALLOC()/XFREE() so heap and + * tag reach every allocator. Mixing aligned and plain (XMALLOC()/XFREE()) + * allocations is not safe. + */ +#ifndef XMALLOCALIGN + #define XMALLOCALIGN(a, s, h, t) wc_MallocAlign(a, s, h, t) + #define XFREEALIGN(p, h, t) wc_FreeAlign((p), (h), (t)) + /* reallocation for aligned allocations */ + #define XREALLOCALIGN(p, align, size, h, t) \ + wc_ReallocAlign(p, align, size, h, t) + /* reallocation with scrubbing of lost blocks for aligned allocations */ + #define XREALLOCALIGN_SCRUBBED(p, align, size, h, t) \ + wc_ReallocAlignScrubbed(p, align, size, h, t) +#endif + +/* reallocation with scrubbing of lost blocks */ #ifndef XREALLOC_SCRUBBED #define XREALLOC_SCRUBBED(p, old_size, size, h, t) \ - wc_xrealloc_scrubbed(p, old_size, size, h, t) -#endif -#ifndef XREALLOCALIGN - #define XREALLOCALIGN(p, align, old_size, size, h, t) \ - wc_xreallocalign(p, align, old_size, size, h, t, 0) -#endif -#ifndef XREALLOCALIGN_SCRUBBED - #define XREALLOCALIGN_SCRUBBED(p, align, old_size, size, h, t) \ - wc_xreallocalign(p, align, old_size, size, h, t, 1) + wc_ReallocScrubbed(p, old_size, size, h, t) #endif #if defined(WOLFSSL_SMALL_STACK) && defined(WC_NO_CONSTRUCTORS) @@ -1137,7 +1115,7 @@ enum { } #define WC_ALLOC_HEAP_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ for (idx##VAR_NAME=0; idx##VAR_NAME<(VAR_ITEMS); idx##VAR_NAME++) { \ - (VAR_NAME)[idx##VAR_NAME] = (VAR_TYPE*)XMEMALIGN(VAR_ALIGN, VAR_SIZE, (HEAP), \ + (VAR_NAME)[idx##VAR_NAME] = (VAR_TYPE*)XMALLOCALIGN(VAR_ALIGN, VAR_SIZE, (HEAP), \ DYNAMIC_TYPE_TMP_BUFFER); \ if ((VAR_NAME)[idx##VAR_NAME] == NULL) { \ for (inner_idx_##VAR_NAME = 0; \ @@ -1195,6 +1173,12 @@ enum { idx##VAR_NAME = 0; \ } +/* _ALIGNED declarations: VAR_ALIGN must be a positive power of 2, rejected at + * compile time in both the heap and stack variants. */ +#define WC_ALIGN_ASSERT(VAR_ALIGN) \ + wc_static_assert(((VAR_ALIGN) > 0) && \ + (((VAR_ALIGN) & ((VAR_ALIGN) - 1)) == 0)) + #if defined(WOLFSSL_SMALL_STACK) #define WC_DECLARE_VAR_IS_HEAP_ALLOC #define WC_DECLARE_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) \ @@ -1206,6 +1190,7 @@ enum { * alignment, so this holds naturally unless VAR_ALIGN exceeds VAR_TYPE's * intrinsic alignment. */ #define WC_DECLARE_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ + WC_ALIGN_ASSERT(VAR_ALIGN); \ VAR_TYPE* VAR_NAME = NULL #define WC_VAR_OK(VAR_NAME) ((VAR_NAME) != NULL) @@ -1218,10 +1203,10 @@ enum { } \ } while (0) - /* NOTA BENE: _ALIGNED allocs must have matching _ALIGNED frees. */ + /* _ALIGNED allocs must have matching _ALIGNED frees. */ #define WC_ALLOC_VAR_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP) \ do { \ - (VAR_NAME) = (VAR_TYPE*)XMEMALIGN(VAR_ALIGN, sizeof(VAR_TYPE) * (VAR_SIZE), \ + (VAR_NAME) = (VAR_TYPE*)XMALLOCALIGN(VAR_ALIGN, sizeof(VAR_TYPE) * (VAR_SIZE), \ (HEAP), DYNAMIC_TYPE_WOLF_BIGINT); \ if ((VAR_NAME) == NULL) { \ WC_ALLOC_DO_ON_FAILURE(); \ @@ -1236,10 +1221,10 @@ enum { } \ } while (0) - /* NOTA BENE: _ALIGNED allocs must have matching _ALIGNED frees. */ + /* _ALIGNED allocs must have matching _ALIGNED frees. */ #define WC_ALLOC_VAR_ALIGNED_EX(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP, TY, ONFAIL)\ do { \ - (VAR_NAME) = (VAR_TYPE*)XMEMALIGN(VAR_ALIGN, sizeof(VAR_TYPE) * (VAR_SIZE), \ + (VAR_NAME) = (VAR_TYPE*)XMALLOCALIGN(VAR_ALIGN, sizeof(VAR_TYPE) * (VAR_SIZE), \ (HEAP), TY); \ if ((VAR_NAME) == NULL) { \ ONFAIL; \ @@ -1254,7 +1239,7 @@ enum { } \ } while (0) - /* NOTA BENE: _ALIGNED allocs must have matching _ALIGNED frees. */ + /* _ALIGNED allocs must have matching _ALIGNED frees. */ #define WC_CALLOC_VAR_ALIGNED_EX(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP, TY, ONFAIL)\ do { \ WC_ALLOC_VAR_ALIGNED_EX(VAR_NAME, VAR_TYPE, VAR_ALIGN, VAR_SIZE, HEAP, TY, ONFAIL);\ @@ -1290,6 +1275,7 @@ enum { #define WC_DECLARE_ARRAY_ALIGNED(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_ALIGN, VAR_SIZE, HEAP) \ wc_static_assert(((VAR_SIZE) % sizeof(VAR_TYPE)) == 0); \ wc_static_assert(((VAR_SIZE) % (VAR_ALIGN)) == 0); \ + WC_ALIGN_ASSERT(VAR_ALIGN); \ WC_DECLARE_HEAP_ARRAY(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE, HEAP) #define WC_ARRAY_ARG(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE) \ WC_HEAP_ARRAY_ARG(VAR_NAME, VAR_TYPE, VAR_ITEMS, VAR_SIZE) @@ -1309,17 +1295,18 @@ enum { #else #undef WC_DECLARE_VAR_IS_HEAP_ALLOC /* Stack _ALIGNED variants rely on XALIGNED(), so unlike the heap - * (XMEMALIGN()) variants, alignment is only as good as the compiler's: + * (XMALLOCALIGN()) variants, alignment is only as good as the compiler's: * XALIGNED() is a null expansion on compilers without a known alignment * attribute (e.g. Open Watcom), giving no alignment guarantee there. IAR * ARM cannot align stack variables beyond 8 bytes (see WOLFSSL_ALIGN()), * so larger requests are rejected at compile time. */ #if defined(__ICCARM__) #define WC_STACK_ALIGN_ASSERT(VAR_ALIGN) \ + WC_ALIGN_ASSERT(VAR_ALIGN); \ wc_static_assert((VAR_ALIGN) <= 8) #else #define WC_STACK_ALIGN_ASSERT(VAR_ALIGN) \ - wc_static_assert((VAR_ALIGN) > 0) + WC_ALIGN_ASSERT(VAR_ALIGN) #endif #define WC_DECLARE_VAR(VAR_NAME, VAR_TYPE, VAR_SIZE, HEAP) \ VAR_TYPE VAR_NAME[VAR_SIZE] @@ -1372,7 +1359,7 @@ enum { /* nothing to free, its stack */ #endif -#if (defined(HAVE_FIPS) && FIPS_VERSION3_LT(7,0,0)) || defined(HAVE_SELFTEST) +#if defined(HAVE_FIPS) || defined(HAVE_SELFTEST) /* These are here for the FIPS code that can't be changed. * New definitions don't need to be added here. */ #define DECLARE_VAR WC_DECLARE_VAR From 05c9fbe3d3165564d2680628c8a11fcd1014489a Mon Sep 17 00:00:00 2001 From: Aidan Keefe Date: Fri, 30 Oct 2026 16:34:00 -0600 Subject: [PATCH 4/5] changed REALLOCSCRUBBED --- doc/dox_comments/header_files/types.h | 7 +++++-- linuxkm/linuxkm_wc_port.h | 13 +++++++++++++ wolfcrypt/test/test.c | 8 ++++---- wolfssl/wolfcrypt/types.h | 4 ++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/doc/dox_comments/header_files/types.h b/doc/dox_comments/header_files/types.h index 457c356e030..848797ac7ba 100644 --- a/doc/dox_comments/header_files/types.h +++ b/doc/dox_comments/header_files/types.h @@ -249,6 +249,9 @@ void* XREALLOCALIGN(void *p, size_t a, size_t n, void* heap, int type); \param p NULL, or memory from XMALLOCALIGN/XREALLOCALIGN* \param a alignment, as for XMALLOCALIGN + \param old_size current size of p in bytes. Unused by the default + implementation, which tracks the size itself, but available to + overrides that need it. Must not exceed the real allocation size. \param n new size in bytes \param h pointer to the heap to use \param t memory allocation type for user hints. See enum in types.h @@ -256,8 +259,8 @@ void* XREALLOCALIGN(void *p, size_t a, size_t n, void* heap, int type); \sa XREALLOCALIGN \sa wc_ReallocAlignScrubbed */ -void* XREALLOCALIGN_SCRUBBED(void *p, size_t a, size_t n, void* heap, - int type); +void* XREALLOCALIGN_SCRUBBED(void *p, size_t a, size_t old_size, size_t n, + void* heap, int type); /*! \ingroup Memory diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index 8b2abcc2252..ca32f0213de 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -2422,6 +2422,19 @@ #endif /* !XMALLOC_USER && !XMALLOC_OVERRIDE */ #endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) +#ifndef XMALLOCALIGN + #define XMALLOCALIGN(a, s, h, t) ((void)(a), (void)(h), (void)(t), malloc(s)) + #define XFREEALIGN(p, h, t) do { (void)(h); (void)(t); free(p); } while (0) + /* reallocation for aligned allocations */ + #define XREALLOCALIGN(p, align, size, h, t) \ + ((void)(align), (void)(h), (void)(t), realloc(p, size)) + /* reallocation with scrubbing of lost blocks for aligned allocations */ + #define XREALLOCALIGN_SCRUBBED(p, align, old_size, size, h, t) \ + ((void)(align), wc_ReallocScrubbed(p, old_size, size, h, t)) +#endif /* XMALLOCALIGN */ +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) */ + #include #ifndef INT32_MAX diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 806a683df5e..e53ade90844 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -38009,7 +38009,7 @@ static wc_test_ret_t memalign_realloc_test(void) p[i] = (byte)i; /* scrubbed grow: contents and alignment preserved. */ - q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 1600U, HEAP_HINT, + q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 300U, 1600U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (q == NULL) { /* original intact on failure, per contract. */ @@ -38133,7 +38133,7 @@ static wc_test_ret_t memalign_realloc_test(void) p = (byte*)XMALLOCALIGN(64U, 128U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (p == NULL) return WC_TEST_RET_ENC_EC(MEMORY_E); - q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 0U, HEAP_HINT, + q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 128U, 0U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (q != NULL) { XFREEALIGN(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); @@ -38296,7 +38296,7 @@ static wc_test_ret_t memalign_scrub_test(void) return WC_TEST_RET_ENC_EC(MEMORY_E); XMEMSET(p, MEMALIGN_SCRUB_PATTERN, 300U); wc_MemZero_Add("memalign_scrub_test aligned grow", p, 300U); - q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 1600U, HEAP_HINT, + q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 300U, 1600U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (q == NULL) { XMEMSET(p, 0, 300U); @@ -38311,7 +38311,7 @@ static wc_test_ret_t memalign_scrub_test(void) return WC_TEST_RET_ENC_EC(MEMORY_E); XMEMSET(p, MEMALIGN_SCRUB_PATTERN, 300U); wc_MemZero_Add("memalign_scrub_test aligned zero", p, 300U); - q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 0U, HEAP_HINT, + q = (byte*)XREALLOCALIGN_SCRUBBED(p, 64U, 300U, 0U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (q != NULL) { XFREEALIGN(q, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); diff --git a/wolfssl/wolfcrypt/types.h b/wolfssl/wolfcrypt/types.h index 31476a299c5..ff4b4979566 100644 --- a/wolfssl/wolfcrypt/types.h +++ b/wolfssl/wolfcrypt/types.h @@ -974,8 +974,8 @@ enum { #define XREALLOCALIGN(p, align, size, h, t) \ wc_ReallocAlign(p, align, size, h, t) /* reallocation with scrubbing of lost blocks for aligned allocations */ - #define XREALLOCALIGN_SCRUBBED(p, align, size, h, t) \ - wc_ReallocAlignScrubbed(p, align, size, h, t) + #define XREALLOCALIGN_SCRUBBED(p, align, old_size, size, h, t) \ + ((void)old_size, wc_ReallocAlignScrubbed(p, align, size, h, t)) #endif /* reallocation with scrubbing of lost blocks */ From 4d2e357ec275ddb185960526c8b88f5f5fc8eada Mon Sep 17 00:00:00 2001 From: Aidan Keefe Date: Fri, 30 Oct 2026 17:02:16 -0600 Subject: [PATCH 5/5] always compile memory.c --- cmake/functions.cmake | 8 +++++--- src/include.am | 6 ++++-- wolfcrypt/test/test.c | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cmake/functions.cmake b/cmake/functions.cmake index 65bf4dbe1d2..8e192a13bdb 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -976,9 +976,11 @@ function(generate_lib_src_list LIB_SOURCES) wolfcrypt/src/wolfmath.c) endif() - if(BUILD_MEMORY) - list(APPEND LIB_SOURCES wolfcrypt/src/memory.c) - endif() + # memory.c also holds the aligned/scrubbing allocators (wc_MallocAlign(), + # wc_ReallocScrubbed(), ...) that types.h binds unconditionally, so it is + # built even without memory callbacks; the callback code inside it is + # gated on USE_WOLFSSL_MEMORY. + list(APPEND LIB_SOURCES wolfcrypt/src/memory.c) if(NOT BUILD_FIPS_RAND) if(NOT BUILD_FIPS_V2 AND BUILD_DH) diff --git a/src/include.am b/src/include.am index efa55645d83..657ce3c850a 100644 --- a/src/include.am +++ b/src/include.am @@ -1983,9 +1983,11 @@ endif !BUILD_DO178 src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wolfmath.c endif !BUILD_FIPS_RAND -if BUILD_MEMORY +# memory.c also holds the aligned/scrubbing allocators (wc_MallocAlign(), +# wc_ReallocScrubbed(), ...) that types.h binds unconditionally, so it is +# built even without memory callbacks; the callback code inside it is +# gated on USE_WOLFSSL_MEMORY. src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/memory.c -endif if !BUILD_FIPS_RAND if !BUILD_FIPS_V2_PLUS diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index e53ade90844..c601732e74a 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -38061,7 +38061,7 @@ static wc_test_ret_t memalign_realloc_test(void) /* unscrubbed grow well past any allocator slack: contents and alignment * preserved, and only the old size is copied (no over-read). */ if (ret == 0) { - q = (byte*)XREALLOCALIGN(p, 64U, 4096U, HEAP_HINT, + q = (byte*)XREALLOCALIGN(p, 64U, 2048U, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); if (q == NULL) ret = WC_TEST_RET_ENC_EC(MEMORY_E); /* p intact on failure */ @@ -38075,7 +38075,7 @@ static wc_test_ret_t memalign_realloc_test(void) } /* utilize the grown extent. */ if (ret == 0) - XMEMSET(p + 128, 0xA5, 4096U - 128U); + XMEMSET(p + 128, 0xA5, 2048U - 128U); } } XFREEALIGN(p, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);