softhsm: fix build with OpenSSL 4.0

do_compile fails with:

  OSSLUtil.cpp:188:46: error: invalid use of incomplete type
  'ASN1_PRINTABLESTRING' {aka 'struct asn1_string_st'}

Backport the upstream fix. It was written for a memory leak, but it
moves to the ASN1_STRING accessors, which is also what OpenSSL 4 needs.

Signed-off-by: Alper Ak <alperyasinak1@gmail.com>
Signed-off-by: Khem Raj <khem.raj@oss.qualcomm.com>
This commit is contained in:
Alper Ak
2026-09-11 07:07:58 -07:00
committed by Khem Raj
parent 6b7942fa8c
commit 11066da803
2 changed files with 90 additions and 0 deletions
@@ -0,0 +1,89 @@
From: =?UTF-8?q?Ma=C5=82gorzata=20Olsz=C3=B3wka?= <Malgorzata.Olszowka@stunnel.org>
Date: Sun, 3 May 2026 13:00:07 +0200
Subject: [PATCH] Fix memory leak in EdDSA PrintableString handling (#858)
ASN1_STRING is opaque in OpenSSL 4, so curve_name->data no longer
compiles. Use the accessors and free the string on every path.
Upstream-Status: Backport [https://github.com/softhsm/SoftHSMv2/commit/d23ea09d318c03c033420d065f1c64b019cc94ed]
Signed-off-by: Alper Ak <alperyasinak1@gmail.com>
---
src/lib/crypto/OSSLUtil.cpp | 45 ++++++++++++++++++++++++++-----------
1 file changed, 32 insertions(+), 13 deletions(-)
--- a/src/lib/crypto/OSSLUtil.cpp
+++ b/src/lib/crypto/OSSLUtil.cpp
@@ -160,12 +160,14 @@
// Convert a ByteString to an OpenSSL EVP_PKEY id
int OSSL::byteString2oid(const ByteString& byteString)
{
- ASN1_OBJECT *oid = NULL;
- ASN1_PRINTABLESTRING *curve_name = NULL;
+ ASN1_OBJECT *oid;
+ ASN1_PRINTABLESTRING *curve_name;
const unsigned char *p = byteString.const_byte_str();
const unsigned char *pp = p;
+ const unsigned char *data;
long length;
- int tag, pclass;
+ int tag, pclass, data_len;
+ int nid = NID_undef;
ASN1_get_object(&pp, &length, &tag, &pclass, byteString.size());
if (pclass == V_ASN1_UNIVERSAL && tag == V_ASN1_OBJECT)
@@ -178,34 +180,49 @@
return NID_undef;
}
- return OBJ_obj2nid(oid);
+ nid = OBJ_obj2nid(oid);
+ ASN1_OBJECT_free(oid);
}
else if (pclass == V_ASN1_UNIVERSAL && tag == V_ASN1_PRINTABLESTRING)
{
/* The final PKCS#11 3.0 expects curve name encoded as PrintableString */
curve_name = d2i_ASN1_PRINTABLESTRING(NULL, &p, byteString.size());
- if (strcmp((char *)curve_name->data, "edwards25519") == 0)
+ if (curve_name == NULL)
{
+ return NID_undef;
+ }
+
+ data = ASN1_STRING_get0_data(curve_name);
+ data_len = ASN1_STRING_length(curve_name);
+
+ if (data_len == 12 && memcmp(data, "edwards25519", data_len) == 0)
+ {
+ ASN1_PRINTABLESTRING_free(curve_name);
return EVP_PKEY_ED25519;
}
- if (strcmp((char *)curve_name->data, "curve25519") == 0)
+ if (data_len == 10 && memcmp(data, "curve25519", 10) == 0)
{
+ ASN1_PRINTABLESTRING_free(curve_name);
return EVP_PKEY_X25519;
}
- if (strcmp((char *)curve_name->data, "edwards448") == 0)
+ if (data_len == 10 && memcmp(data, "edwards448", 10) == 0)
{
+ ASN1_PRINTABLESTRING_free(curve_name);
return EVP_PKEY_ED448;
}
- if (strcmp((char *)curve_name->data, "curve448") == 0)
+ if (data_len == 8 && memcmp(data, "curve448", 8) == 0)
{
+ ASN1_PRINTABLESTRING_free(curve_name);
return EVP_PKEY_X448;
}
+
+ ASN1_PRINTABLESTRING_free(curve_name);
}
- return NID_undef;
+ return nid;
}
#endif
@@ -7,6 +7,7 @@ DEPENDS = "sqlite3"
SRC_URI = "git://github.com/softhsm/SoftHSMv2.git;protocol=https;branch=main;tag=${PV} \
file://0002-Prevent-accessing-of-global-c-objects-once-they-are-.patch \
file://0003-Fix-memory-leak-in-EdDSA-PrintableString-handling-858.patch \
"
SRCREV = "13e6e86b83748fef74046dbf0c91f664b7acc1c3"