VirtualBox

source: vbox/trunk/src/libs/openssl-3.4.1/crypto/rsa/rsa_pss.c

Last change on this file was 109052, checked in by vboxsync, 5 weeks ago

openssl-3.4.1: Applied our changes, regenerated files, added missing files and functions. This time with a three way merge. ​bugref:10890
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 12.6 KB
Line 
1/*
2 * Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved.
3 *
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
8 */
9
10/*
11 * RSA low level APIs are deprecated for public use, but still ok for
12 * internal use.
13 */
14#include "internal/deprecated.h"
15
16#include <stdio.h>
17#include "internal/cryptlib.h"
18#include <openssl/bn.h>
19#include <openssl/rsa.h>
20#include <openssl/evp.h>
21#include <openssl/rand.h>
22#include <openssl/sha.h>
23#include "rsa_local.h"
24
25static const unsigned char zeroes[] = { 0, 0, 0, 0, 0, 0, 0, 0 };
26
27#if defined(_MSC_VER) && defined(_ARM_)
28# pragma optimize("g", off)
29#endif
30
31int RSA_verify_PKCS1_PSS(RSA *rsa, const unsigned char *mHash,
32 const EVP_MD *Hash, const unsigned char *EM,
33 int sLen)
34{
35 return RSA_verify_PKCS1_PSS_mgf1(rsa, mHash, Hash, NULL, EM, sLen);
36}
37
38int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
39 const EVP_MD *Hash, const EVP_MD *mgf1Hash,
40 const unsigned char *EM, int sLen)
41{
42 return ossl_rsa_verify_PKCS1_PSS_mgf1(rsa, mHash, Hash, mgf1Hash, EM, &sLen);
43}
44
45int ossl_rsa_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
46 const EVP_MD *Hash, const EVP_MD *mgf1Hash,
47 const unsigned char *EM, int *sLenOut)
48{
49 int i;
50 int ret = 0;
51 int sLen = *sLenOut;
52 int hLen, maskedDBLen, MSBits, emLen;
53 const unsigned char *H;
54 unsigned char *DB = NULL;
55 EVP_MD_CTX *ctx = EVP_MD_CTX_new();
56 unsigned char H_[EVP_MAX_MD_SIZE];
57
58 if (ctx == NULL)
59 goto err;
60
61 if (mgf1Hash == NULL)
62 mgf1Hash = Hash;
63
64 hLen = EVP_MD_get_size(Hash);
65 if (hLen <= 0)
66 goto err;
67 /*-
68 * Negative sLen has special meanings:
69 * -1 sLen == hLen
70 * -2 salt length is autorecovered from signature
71 * -3 salt length is maximized
72 * -4 salt length is autorecovered from signature
73 * -N reserved
74 */
75 if (sLen == RSA_PSS_SALTLEN_DIGEST) {
76 sLen = hLen;
77 } else if (sLen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
78 ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED);
79 goto err;
80 }
81
82 MSBits = (BN_num_bits(rsa->n) - 1) & 0x7;
83 emLen = RSA_size(rsa);
84 if (EM[0] & (0xFF << MSBits)) {
85 ERR_raise(ERR_LIB_RSA, RSA_R_FIRST_OCTET_INVALID);
86 goto err;
87 }
88 if (MSBits == 0) {
89 EM++;
90 emLen--;
91 }
92 if (emLen < hLen + 2) {
93 ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE);
94 goto err;
95 }
96 if (sLen == RSA_PSS_SALTLEN_MAX) {
97 sLen = emLen - hLen - 2;
98 } else if (sLen > emLen - hLen - 2) { /* sLen can be small negative */
99 ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE);
100 goto err;
101 }
102 if (EM[emLen - 1] != 0xbc) {
103 ERR_raise(ERR_LIB_RSA, RSA_R_LAST_OCTET_INVALID);
104 goto err;
105 }
106 maskedDBLen = emLen - hLen - 1;
107 H = EM + maskedDBLen;
108 DB = OPENSSL_malloc(maskedDBLen);
109 if (DB == NULL)
110 goto err;
111 if (PKCS1_MGF1(DB, maskedDBLen, H, hLen, mgf1Hash) < 0)
112 goto err;
113 for (i = 0; i < maskedDBLen; i++)
114 DB[i] ^= EM[i];
115 if (MSBits)
116 DB[0] &= 0xFF >> (8 - MSBits);
117 for (i = 0; DB[i] == 0 && i < (maskedDBLen - 1); i++) ;
118 if (DB[i++] != 0x1) {
119 ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_RECOVERY_FAILED);
120 goto err;
121 }
122 if (sLen != RSA_PSS_SALTLEN_AUTO
123 && sLen != RSA_PSS_SALTLEN_AUTO_DIGEST_MAX
124 && (maskedDBLen - i) != sLen) {
125 ERR_raise_data(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED,
126 "expected: %d retrieved: %d", sLen,
127 maskedDBLen - i);
128 goto err;
129 } else {
130 sLen = maskedDBLen - i;
131 }
132 if (!EVP_DigestInit_ex(ctx, Hash, NULL)
133 || !EVP_DigestUpdate(ctx, zeroes, sizeof(zeroes))
134 || !EVP_DigestUpdate(ctx, mHash, hLen))
135 goto err;
136 if (sLen != 0) {
137 if (!EVP_DigestUpdate(ctx, DB + i, sLen))
138 goto err;
139 }
140 if (!EVP_DigestFinal_ex(ctx, H_, NULL))
141 goto err;
142 if (memcmp(H_, H, hLen)) {
143 ERR_raise(ERR_LIB_RSA, RSA_R_BAD_SIGNATURE);
144 ret = 0;
145 } else {
146 ret = 1;
147 }
148
149 *sLenOut = sLen;
150 err:
151 OPENSSL_free(DB);
152 EVP_MD_CTX_free(ctx);
153
154 return ret;
155
156}
157
158int RSA_padding_add_PKCS1_PSS(RSA *rsa, unsigned char *EM,
159 const unsigned char *mHash,
160 const EVP_MD *Hash, int sLen)
161{
162 return RSA_padding_add_PKCS1_PSS_mgf1(rsa, EM, mHash, Hash, NULL, sLen);
163}
164
165int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
166 const unsigned char *mHash,
167 const EVP_MD *Hash, const EVP_MD *mgf1Hash,
168 int sLen)
169{
170 return ossl_rsa_padding_add_PKCS1_PSS_mgf1(rsa, EM, mHash, Hash, mgf1Hash, &sLen);
171}
172
173int ossl_rsa_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
174 const unsigned char *mHash,
175 const EVP_MD *Hash, const EVP_MD *mgf1Hash,
176 int *sLenOut)
177{
178 int i;
179 int ret = 0;
180 int sLen = *sLenOut;
181 int hLen, maskedDBLen, MSBits, emLen;
182 unsigned char *H, *salt = NULL, *p;
183 EVP_MD_CTX *ctx = NULL;
184 int sLenMax = -1;
185
186 if (mgf1Hash == NULL)
187 mgf1Hash = Hash;
188
189 hLen = EVP_MD_get_size(Hash);
190 if (hLen <= 0)
191 goto err;
192 /*-
193 * Negative sLen has special meanings:
194 * -1 sLen == hLen
195 * -2 salt length is maximized
196 * -3 same as above (on signing)
197 * -4 salt length is min(hLen, maximum salt length)
198 * -N reserved
199 */
200 /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection
201 * 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the
202 * salt (sLen) shall satisfy 0 <= sLen <= hLen, where hLen is the length of
203 * the hash function output block (in bytes)."
204 *
205 * Provide a way to use at most the digest length, so that the default does
206 * not violate FIPS 186-4. */
207 if (sLen == RSA_PSS_SALTLEN_DIGEST) {
208 sLen = hLen;
209 } else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN
210 || sLen == RSA_PSS_SALTLEN_AUTO) {
211 sLen = RSA_PSS_SALTLEN_MAX;
212 } else if (sLen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
213 sLen = RSA_PSS_SALTLEN_MAX;
214 sLenMax = hLen;
215 } else if (sLen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
216 ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED);
217 goto err;
218 }
219
220 MSBits = (BN_num_bits(rsa->n) - 1) & 0x7;
221 emLen = RSA_size(rsa);
222 if (MSBits == 0) {
223 *EM++ = 0;
224 emLen--;
225 }
226 if (emLen < hLen + 2) {
227 ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
228 goto err;
229 }
230 if (sLen == RSA_PSS_SALTLEN_MAX) {
231 sLen = emLen - hLen - 2;
232 if (sLenMax >= 0 && sLen > sLenMax)
233 sLen = sLenMax;
234 } else if (sLen > emLen - hLen - 2) {
235 ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
236 goto err;
237 }
238 if (sLen > 0) {
239 salt = OPENSSL_malloc(sLen);
240 if (salt == NULL)
241 goto err;
242 if (RAND_bytes_ex(rsa->libctx, salt, sLen, 0) <= 0)
243 goto err;
244 }
245 maskedDBLen = emLen - hLen - 1;
246 H = EM + maskedDBLen;
247 ctx = EVP_MD_CTX_new();
248 if (ctx == NULL)
249 goto err;
250 if (!EVP_DigestInit_ex(ctx, Hash, NULL)
251 || !EVP_DigestUpdate(ctx, zeroes, sizeof(zeroes))
252 || !EVP_DigestUpdate(ctx, mHash, hLen))
253 goto err;
254 if (sLen && !EVP_DigestUpdate(ctx, salt, sLen))
255 goto err;
256 if (!EVP_DigestFinal_ex(ctx, H, NULL))
257 goto err;
258
259 /* Generate dbMask in place then perform XOR on it */
260 if (PKCS1_MGF1(EM, maskedDBLen, H, hLen, mgf1Hash))
261 goto err;
262
263 p = EM;
264
265 /*
266 * Initial PS XORs with all zeroes which is a NOP so just update pointer.
267 * Note from a test above this value is guaranteed to be non-negative.
268 */
269 p += emLen - sLen - hLen - 2;
270 *p++ ^= 0x1;
271 if (sLen > 0) {
272 for (i = 0; i < sLen; i++)
273 *p++ ^= salt[i];
274 }
275 if (MSBits)
276 EM[0] &= 0xFF >> (8 - MSBits);
277
278 /* H is already in place so just set final 0xbc */
279
280 EM[emLen - 1] = 0xbc;
281
282 ret = 1;
283
284 *sLenOut = sLen;
285 err:
286 EVP_MD_CTX_free(ctx);
287 OPENSSL_clear_free(salt, (size_t)sLen); /* salt != NULL implies sLen > 0 */
288
289 return ret;
290
291}
292
293/*
294 * The defaults for PSS restrictions are defined in RFC 8017, A.2.3 RSASSA-PSS
295 * (https://tools.ietf.org/html/rfc8017#appendix-A.2.3):
296 *
297 * If the default values of the hashAlgorithm, maskGenAlgorithm, and
298 * trailerField fields of RSASSA-PSS-params are used, then the algorithm
299 * identifier will have the following value:
300 *
301 * rSASSA-PSS-Default-Identifier RSASSA-AlgorithmIdentifier ::= {
302 * algorithm id-RSASSA-PSS,
303 * parameters RSASSA-PSS-params : {
304 * hashAlgorithm sha1,
305 * maskGenAlgorithm mgf1SHA1,
306 * saltLength 20,
307 * trailerField trailerFieldBC
308 * }
309 * }
310 *
311 * RSASSA-AlgorithmIdentifier ::= AlgorithmIdentifier {
312 * {PKCS1Algorithms}
313 * }
314 */
315static const RSA_PSS_PARAMS_30 default_RSASSA_PSS_params = {
316 NID_sha1, /* default hashAlgorithm */
317 {
318 NID_mgf1, /* default maskGenAlgorithm */
319 NID_sha1 /* default MGF1 hash */
320 },
321 20, /* default saltLength */
322 1 /* default trailerField (0xBC) */
323};
324
325int ossl_rsa_pss_params_30_set_defaults(RSA_PSS_PARAMS_30 *rsa_pss_params)
326{
327 if (rsa_pss_params == NULL)
328 return 0;
329 *rsa_pss_params = default_RSASSA_PSS_params;
330 return 1;
331}
332
333int ossl_rsa_pss_params_30_is_unrestricted(const RSA_PSS_PARAMS_30 *rsa_pss_params)
334{
335 static RSA_PSS_PARAMS_30 pss_params_cmp = { 0, };
336
337 return rsa_pss_params == NULL
338 || memcmp(rsa_pss_params, &pss_params_cmp,
339 sizeof(*rsa_pss_params)) == 0;
340}
341
342int ossl_rsa_pss_params_30_copy(RSA_PSS_PARAMS_30 *to,
343 const RSA_PSS_PARAMS_30 *from)
344{
345 memcpy(to, from, sizeof(*to));
346 return 1;
347}
348
349int ossl_rsa_pss_params_30_set_hashalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
350 int hashalg_nid)
351{
352 if (rsa_pss_params == NULL)
353 return 0;
354 rsa_pss_params->hash_algorithm_nid = hashalg_nid;
355 return 1;
356}
357
358int ossl_rsa_pss_params_30_set_maskgenhashalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
359 int maskgenhashalg_nid)
360{
361 if (rsa_pss_params == NULL)
362 return 0;
363 rsa_pss_params->mask_gen.hash_algorithm_nid = maskgenhashalg_nid;
364 return 1;
365}
366
367int ossl_rsa_pss_params_30_set_saltlen(RSA_PSS_PARAMS_30 *rsa_pss_params,
368 int saltlen)
369{
370 if (rsa_pss_params == NULL)
371 return 0;
372 rsa_pss_params->salt_len = saltlen;
373 return 1;
374}
375
376int ossl_rsa_pss_params_30_set_trailerfield(RSA_PSS_PARAMS_30 *rsa_pss_params,
377 int trailerfield)
378{
379 if (rsa_pss_params == NULL)
380 return 0;
381 rsa_pss_params->trailer_field = trailerfield;
382 return 1;
383}
384
385int ossl_rsa_pss_params_30_hashalg(const RSA_PSS_PARAMS_30 *rsa_pss_params)
386{
387 if (rsa_pss_params == NULL)
388 return default_RSASSA_PSS_params.hash_algorithm_nid;
389 return rsa_pss_params->hash_algorithm_nid;
390}
391
392int ossl_rsa_pss_params_30_maskgenalg(const RSA_PSS_PARAMS_30 *rsa_pss_params)
393{
394 if (rsa_pss_params == NULL)
395 return default_RSASSA_PSS_params.mask_gen.algorithm_nid;
396 return rsa_pss_params->mask_gen.algorithm_nid;
397}
398
399int ossl_rsa_pss_params_30_maskgenhashalg(const RSA_PSS_PARAMS_30 *rsa_pss_params)
400{
401 if (rsa_pss_params == NULL)
402 return default_RSASSA_PSS_params.hash_algorithm_nid;
403 return rsa_pss_params->mask_gen.hash_algorithm_nid;
404}
405
406int ossl_rsa_pss_params_30_saltlen(const RSA_PSS_PARAMS_30 *rsa_pss_params)
407{
408 if (rsa_pss_params == NULL)
409 return default_RSASSA_PSS_params.salt_len;
410 return rsa_pss_params->salt_len;
411}
412
413int ossl_rsa_pss_params_30_trailerfield(const RSA_PSS_PARAMS_30 *rsa_pss_params)
414{
415 if (rsa_pss_params == NULL)
416 return default_RSASSA_PSS_params.trailer_field;
417 return rsa_pss_params->trailer_field;
418}
419
420#if defined(_MSC_VER)
421# pragma optimize("",on)
422#endif
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette