ssl-openssl.cpp@ 74300

Last change on this file since 74300 was 74300, checked in by vboxsync, 7 years ago
IPRT/crypto: Wrapped SSL, alleged RC4, and RSA key generation. Added methods for quering someRSA key components. Exposed big numbers, x509 and asn1 APIs as stable. bugref:9246 [openssl 1.0.x build fix]
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 15.1 KB

Line
1	/** @file
2	* IPRT - Crypto - Secure Socket Layer (SSL) / Transport Security Layer (TLS).
3	*/
4
5	/*
6	* Copyright (C) 2018 Oracle Corporation
7	*
8	* This file is part of VirtualBox Open Source Edition (OSE), as
9	* available from http://www.215389.xyz. This file is free software;
10	* you can redistribute it and/or modify it under the terms of the GNU
11	* General Public License (GPL) as published by the Free Software
12	* Foundation, in version 2 as it comes in the "COPYING" file of the
13	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15	*
16	* The contents of this file may alternatively be used under the terms
17	* of the Common Development and Distribution License Version 1.0
18	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19	* VirtualBox OSE distribution, in which case the provisions of the
20	* CDDL are applicable instead of those of the GPL.
21	*
22	* You may elect to license modified versions of this file under the
23	* terms and conditions of either the GPL or the CDDL or both.
24	*/
25
26	#ifdef IPRT_WITH_OPENSSL
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	# include "internal/iprt.h"
32	# include <iprt/crypto/ssl.h>
33
34	# include <iprt/asm.h>
35	# include <iprt/assert.h>
36	# include <iprt/file.h>
37	# include <iprt/mem.h>
38	# include <iprt/string.h>
39
40	# include "internal/magics.h"
41
42	# include "internal/iprt-openssl.h"
43	# include <openssl/ssl.h>
44	# include <openssl/tls1.h>
45
46
47	/*********************************************************************************************************************************
48	* Header Files *
49	*********************************************************************************************************************************/
50	/**
51	* SSL instance data for OpenSSL.
52	*/
53	typedef struct RTCRSSLINT
54	{
55	/** Magic value (RTCRSSLINT_MAGIC). */
56	uint32_t u32Magic;
57	/** Reference count. */
58	uint32_t volatile cRefs;
59	/** The SSL context. */
60	SSL_CTX *pCtx;
61	} RTCRSSLINT;
62
63	/**
64	* SSL session instance data for OpenSSL.
65	*/
66	typedef struct RTCRSSLSESSIONINT
67	{
68	/** Magic value (RTCRSSLSESSIONINT_MAGIC). */
69	uint32_t u32Magic;
70	/** Reference count. */
71	uint32_t volatile cRefs;
72	/** RTCRSSLSESSION_F_XXX. */
73	uint32_t fFlags;
74
75	/** The SSL instance. */
76	SSL *pSsl;
77	/** The socket BIO instance. */
78	BIO *pBio;
79	} RTCRSSLSESSIONINT;
80
81
82
83	RTDECL(int) RTCrSslCreate(PRTCRSSL phSsl, uint32_t fFlags)
84	{
85	AssertPtr(phSsl);
86	*phSsl = NIL_RTCRSSL;
87	AssertReturn(!fFlags, VERR_INVALID_FLAGS);
88
89	/*
90	* We aim at TLSv1 or higher here by default.
91	*/
92	# if OPENSSL_VERSION_NUMBER >= 0x10100000
93	const SSL_METHOD *pSslMethod = TLS_method();
94	# elif OPENSSL_VERSION_NUMBER >= 0x10002000
95	const SSL_METHOD *pSslMethod = SSLv23_method();
96	# elif OPENSSL_VERSION_NUMBER >= 0x10000000
97	const SSL_METHOD *pSslMethod = TLSv1_method();
98	# else
99	SSL_METHOD *pSslMethod = TLSv1_method();
100	# endif
101	if (pSslMethod)
102	{
103	RTCRSSLINT pThis = (RTCRSSLINT )RTMemAllocZ(sizeof(*pThis));
104	if (pThis)
105	{
106	pThis->pCtx = SSL_CTX_new(pSslMethod);
107	if (pThis->pCtx)
108	{
109	/* Help with above aim. */
110	# if OPENSSL_VERSION_NUMBER >= 0x10100000
111	if (SSL_CTX_get_min_proto_version(pThis->pCtx) < TLS1_VERSION)
112	SSL_CTX_set_min_proto_version(pThis->pCtx, TLS1_VERSION);
113	# elif OPENSSL_VERSION_NUMBER >= 0x10002000
114	SSL_CTX_set_options(pThis->pCtx, SSL_OP_NO_SSLv2);
115	SSL_CTX_set_options(pThis->pCtx, SSL_OP_NO_SSLv3);
116	# endif
117
118	/*
119	* Complete the instance and return it.
120	*/
121	pThis->u32Magic = RTCRSSLINT_MAGIC;
122	pThis->cRefs = 1;
123
124	*phSsl = pThis;
125	return VINF_SUCCESS;
126	}
127	}
128	return VERR_NO_MEMORY;
129	}
130	return VERR_NOT_SUPPORTED;
131	}
132
133
134	RTDECL(uint32_t) RTCrSslRetain(RTCRSSL hSsl)
135	{
136	RTCRSSLINT *pThis = hSsl;
137	AssertPtrReturn(pThis, UINT32_MAX);
138	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, UINT32_MAX);
139
140	uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
141	Assert(cRefs > 1);
142	Assert(cRefs < 1024);
143	return cRefs;
144	}
145
146
147	/**
148	* Worker for RTCrSslRelease.
149	*/
150	static int rtCrSslDestroy(RTCRSSLINT *pThis)
151	{
152	ASMAtomicWriteU32(&pThis->u32Magic, ~RTCRSSLINT_MAGIC);
153	SSL_CTX_free(pThis->pCtx);
154	pThis->pCtx = NULL;
155	RTMemFree(pThis);
156	return 0;
157	}
158
159
160	RTDECL(uint32_t) RTCrSslRelease(RTCRSSL hSsl)
161	{
162	RTCRSSLINT *pThis = hSsl;
163	if (pThis == NIL_RTCRSSL)
164	return 0;
165	AssertPtrReturn(pThis, UINT32_MAX);
166	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, UINT32_MAX);
167
168	uint32_t cRefs = ASMAtomicDecU32(&pThis->cRefs);
169	Assert(cRefs < 1024);
170	if (cRefs == 0)
171	return rtCrSslDestroy(pThis);
172	return cRefs;
173	}
174
175
176	RTDECL(int) RTCrSslSetCertificateFile(RTCRSSL hSsl, const char *pszFile, uint32_t fFlags)
177	{
178	RTCRSSLINT *pThis = hSsl;
179	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
180	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
181	AssertReturn(!(fFlags & ~RTCRSSL_FILE_F_ASN1), VERR_INVALID_FLAGS);
182
183	int rcOssl = SSL_CTX_use_certificate_file(pThis->pCtx, pszFile,
184	RTCRSSL_FILE_F_ASN1 & fFlags ? SSL_FILETYPE_PEM : SSL_FILETYPE_ASN1);
185	if (rcOssl != 0)
186	return VINF_SUCCESS;
187	return !RTFileExists(pszFile) ? VERR_FILE_NOT_FOUND : VERR_OPEN_FAILED; /** @todo Better status codes */
188	}
189
190
191	RTDECL(int) RTCrSslSetPrivateKeyFile(RTCRSSL hSsl, const char *pszFile, uint32_t fFlags)
192	{
193	RTCRSSLINT *pThis = hSsl;
194	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
195	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
196	AssertReturn(!(fFlags & ~RTCRSSL_FILE_F_ASN1), VERR_INVALID_FLAGS);
197
198	int rcOssl = SSL_CTX_use_PrivateKey_file(pThis->pCtx, pszFile,
199	RTCRSSL_FILE_F_ASN1 & fFlags ? SSL_FILETYPE_PEM : SSL_FILETYPE_ASN1);
200	if (rcOssl != 0)
201	return VINF_SUCCESS;
202	return !RTFileExists(pszFile) ? VERR_FILE_NOT_FOUND : VERR_OPEN_FAILED; /** @todo Better status codes */
203	}
204
205
206	RTDECL(int) RTCrSslLoadTrustedRootCerts(RTCRSSL hSsl, const char pszFile, const char pszDir)
207	{
208	RTCRSSLINT *pThis = hSsl;
209	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
210	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
211
212	int rcOssl = SSL_CTX_load_verify_locations(pThis->pCtx, pszFile, pszDir);
213	if (rcOssl != 0)
214	return VINF_SUCCESS;
215
216	if (pszFile && !RTFileExists(pszFile))
217	return VERR_FILE_NOT_FOUND;
218	return VERR_OPEN_FAILED; /** @todo Better status codes */
219	}
220
221
222	RTDECL(int) RTCrSslSetNoPeerVerify(RTCRSSL hSsl)
223	{
224	RTCRSSLINT *pThis = hSsl;
225	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
226	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
227
228	SSL_CTX_set_verify(pThis->pCtx, SSL_VERIFY_NONE, NULL);
229	return VINF_SUCCESS;
230	}
231
232
233
234	//RTDECL(int) RTCrSslCreateSession(RTCRSSL hSsl, RTSOCKET hSocket, uint32_t fFlags, PRTCRSSLSESSION phSslConn);
235
236	RTDECL(int) RTCrSslCreateSessionForNativeSocket(RTCRSSL hSsl, RTHCINTPTR hNativeSocket, uint32_t fFlags,
237	PRTCRSSLSESSION phSslSession)
238	{
239	/*
240	* Validate input.
241	*/
242	*phSslSession = NIL_RTCRSSLSESSION;
243
244	RTCRSSLINT *pThis = hSsl;
245	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
246	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
247	AssertReturn(!(fFlags & ~RTCRSSLSESSION_F_NON_BLOCKING), VERR_INVALID_FLAGS);
248
249	/*
250	* Create a new session.
251	*/
252	int rc = VERR_NO_MEMORY;
253	RTCRSSLSESSIONINT pSession = (RTCRSSLSESSIONINT )RTMemAllocZ(sizeof(*pSession));
254	if (pSession)
255	{
256	pSession->pSsl = SSL_new(pThis->pCtx);
257	if (pSession->pSsl)
258	{
259	/* Disable read-ahead if non-blocking socket relying on select/poll. */
260	if (fFlags & RTCRSSLSESSION_F_NON_BLOCKING)
261	SSL_set_read_ahead(pSession->pSsl, 0);
262
263	/* Create a wrapper for the socket handle. */
264	pSession->pBio = BIO_new_socket(hNativeSocket, BIO_NOCLOSE);
265	if (pSession->pBio)
266	{
267	# if OPENSSL_VERSION_NUMBER >= 0x10100000
268	BIO_up_ref(pSession->pBio); /* our reference. */
269	# endif
270	SSL_set_bio(pSession->pSsl, pSession->pBio, pSession->pBio);
271
272	/*
273	* Done.
274	*/
275	pSession->cRefs = 1;
276	pSession->u32Magic = RTCRSSLSESSIONINT_MAGIC;
277	*phSslSession = pSession;
278	return VINF_SUCCESS;
279	}
280
281	SSL_free(pSession->pSsl);
282	pSession->pSsl = NULL;
283	}
284	RTMemFree(pThis);
285	}
286	return rc;
287	}
288
289
290	/*********************************************************************************************************************************
291	* Session implementation. *
292	*********************************************************************************************************************************/
293
294	RTDECL(uint32_t) RTCrSslSessionRetain(RTCRSSLSESSION hSslSession)
295	{
296	RTCRSSLSESSIONINT *pThis = hSslSession;
297	AssertPtrReturn(pThis, UINT32_MAX);
298	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, UINT32_MAX);
299
300	uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
301	Assert(cRefs > 1);
302	Assert(cRefs < 1024);
303	return cRefs;
304	}
305
306
307	/**
308	* Worker for RTCrSslRelease.
309	*/
310	static int rtCrSslSessionDestroy(RTCRSSLSESSIONINT *pThis)
311	{
312	ASMAtomicWriteU32(&pThis->u32Magic, ~RTCRSSLSESSIONINT_MAGIC);
313	SSL_free(pThis->pSsl);
314	pThis->pSsl = NULL;
315	# if OPENSSL_VERSION_NUMBER >= 0x10100000
316	BIO_free(pThis->pBio);
317	# endif
318	pThis->pBio = NULL;
319	RTMemFree(pThis);
320	return 0;
321	}
322
323
324	RTDECL(uint32_t) RTCrSslSessionRelease(RTCRSSLSESSION hSslSession)
325	{
326	RTCRSSLSESSIONINT *pThis = hSslSession;
327	if (pThis == NIL_RTCRSSLSESSION)
328	return 0;
329	AssertPtrReturn(pThis, UINT32_MAX);
330	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, UINT32_MAX);
331
332	uint32_t cRefs = ASMAtomicDecU32(&pThis->cRefs);
333	Assert(cRefs < 1024);
334	if (cRefs == 0)
335	return rtCrSslSessionDestroy(pThis);
336	return cRefs;
337	}
338
339
340	RTDECL(int) RTCrSslSessionAccept(RTCRSSLSESSION hSslSession, uint32_t fFlags)
341	{
342	RTCRSSLSESSIONINT *pThis = hSslSession;
343	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
344	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
345	AssertReturn(!fFlags, VERR_INVALID_FLAGS);
346
347	int rcOssl = SSL_accept(pThis->pSsl);
348	if (rcOssl > 0)
349	return VINF_SUCCESS;
350
351	/** @todo better status codes. */
352	if (BIO_should_retry(pThis->pBio))
353	return VERR_TRY_AGAIN;
354	return VERR_NOT_SUPPORTED;
355	}
356
357
358	RTDECL(int) RTCrSslSessionConnect(RTCRSSLSESSION hSslSession, uint32_t fFlags)
359	{
360	RTCRSSLSESSIONINT *pThis = hSslSession;
361	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
362	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
363	AssertReturn(!fFlags, VERR_INVALID_FLAGS);
364
365	int rcOssl = SSL_connect(pThis->pSsl);
366	if (rcOssl > 0)
367	return VINF_SUCCESS;
368
369	/** @todo better status codes. */
370	if (BIO_should_retry(pThis->pBio))
371	return VERR_TRY_AGAIN;
372	return VERR_NOT_SUPPORTED;
373	}
374
375
376	RTDECL(const char *) RTCrSslSessionGetVersion(RTCRSSLSESSION hSslSession)
377	{
378	RTCRSSLSESSIONINT *pThis = hSslSession;
379	AssertPtrReturn(pThis, NULL);
380	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, NULL);
381
382	return SSL_get_version(pThis->pSsl);
383	}
384
385
386	RTDECL(int) RTCrSslSessionGetCertIssuerNameAsString(RTCRSSLSESSION hSslSession, char pszBuf, size_t cbBuf, size_t pcbActual)
387	{
388	RTCRSSLSESSIONINT *pThis = hSslSession;
389	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
390	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
391	AssertPtrNull(pszBuf);
392	AssertPtrNull(pcbActual);
393	if (*pcbActual)
394	pcbActual = 0;
395
396	/*
397	* Get and format the certificate issuer name.
398	*/
399	int rc = VERR_NOT_AVAILABLE;
400	X509 *pCert = SSL_get_certificate(pThis->pSsl);
401	if (pCert)
402	{
403	X509_NAME *pIssuer = X509_get_issuer_name(pCert);
404	if (pIssuer)
405	{
406	char *pszSrc = X509_NAME_oneline(pIssuer, NULL, 0);
407	if (pszSrc)
408	{
409	/*
410	* Copy out the result and free it.
411	*/
412	size_t cbNeeded = strlen(pszSrc) + 1;
413	if (*pcbActual)
414	*pcbActual = cbNeeded;
415
416	if (pszBuf != NULL && cbBuf > 0)
417	{
418	if (cbBuf >= cbNeeded)
419	{
420	memcpy(pszBuf, pszSrc, cbNeeded);
421	rc = VINF_SUCCESS;
422	}
423	else
424	{
425	memcpy(pszBuf, pszSrc, cbBuf - 1);
426	pszBuf[cbBuf - 1] = '\0';
427	rc = VERR_BUFFER_OVERFLOW;
428	}
429	}
430	else
431	rc = VERR_BUFFER_OVERFLOW;
432	OPENSSL_free(pszSrc);
433	}
434	}
435	}
436	return rc;
437	}
438
439
440	RTDECL(bool) RTCrSslSessionPending(RTCRSSLSESSION hSslSession)
441	{
442	RTCRSSLSESSIONINT *pThis = hSslSession;
443	AssertPtrReturn(pThis, true);
444	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, true);
445
446	return SSL_pending(pThis->pSsl) != 0;
447	}
448
449
450	RTDECL(ssize_t) RTCrSslSessionRead(RTCRSSLSESSION hSslSession, void *pvBuf, size_t cbToRead)
451	{
452	RTCRSSLSESSIONINT *pThis = hSslSession;
453	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
454	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
455
456	Assert((size_t)(int)cbToRead == cbToRead);
457
458	int cbActual = SSL_read(pThis->pSsl, pvBuf, (int)cbToRead);
459	if (cbActual > 0)
460	return cbActual;
461	if (BIO_should_retry(pThis->pBio))
462	return VERR_TRY_AGAIN;
463	return VERR_READ_ERROR; /** @todo better status codes. */
464	}
465
466
467	RTDECL(ssize_t) RTCrSslSessionWrite(RTCRSSLSESSION hSslSession, void const *pvBuf, size_t cbToWrite)
468	{
469	RTCRSSLSESSIONINT *pThis = hSslSession;
470	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
471	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
472
473	Assert((size_t)(int)cbToWrite == cbToWrite);
474	Assert(cbToWrite != 0 /* undefined behavior if zero */);
475
476	int cbActual = SSL_write(pThis->pSsl, pvBuf, (int)cbToWrite);
477	if (cbActual > 0)
478	return cbActual;
479	if (BIO_should_retry(pThis->pBio))
480	return VERR_TRY_AGAIN;
481	return VERR_WRITE_ERROR; /** @todo better status codes. */
482	}
483
484	#endif /* IPRT_WITH_OPENSSL */
485

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/crypto/ssl-openssl.cpp@ 74300

Download in other formats: