ssl-openssl.cpp@ 74295

Last change on this file since 74295 was 74295, checked in by vboxsync, 7 years ago
IPRT/crypto: Wrapped SSL, alleged RC4, and RSA key generation. Added methods for quering someRSA key components. Exposed big numbers, x509 and asn1 APIs as stable. bugref:9246
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 14.1 KB

Line
1	/** @file
2	* IPRT - Crypto - Secure Socket Layer (SSL) / Transport Security Layer (TLS).
3	*/
4
5	/*
6	* Copyright (C) 2018 Oracle Corporation
7	*
8	* This file is part of VirtualBox Open Source Edition (OSE), as
9	* available from http://www.215389.xyz. This file is free software;
10	* you can redistribute it and/or modify it under the terms of the GNU
11	* General Public License (GPL) as published by the Free Software
12	* Foundation, in version 2 as it comes in the "COPYING" file of the
13	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15	*
16	* The contents of this file may alternatively be used under the terms
17	* of the Common Development and Distribution License Version 1.0
18	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19	* VirtualBox OSE distribution, in which case the provisions of the
20	* CDDL are applicable instead of those of the GPL.
21	*
22	* You may elect to license modified versions of this file under the
23	* terms and conditions of either the GPL or the CDDL or both.
24	*/
25
26	#ifdef IPRT_WITH_OPENSSL
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#include "internal/iprt.h"
32	#include <iprt/crypto/ssl.h>
33
34	#include <iprt/asm.h>
35	#include <iprt/assert.h>
36	#include <iprt/file.h>
37	#include <iprt/mem.h>
38
39	#include "internal/magics.h"
40
41	#include "internal/iprt-openssl.h"
42	#include <openssl/ssl.h>
43
44
45	/*********************************************************************************************************************************
46	* Header Files *
47	*********************************************************************************************************************************/
48	/**
49	* SSL instance data for OpenSSL.
50	*/
51	typedef struct RTCRSSLINT
52	{
53	/** Magic value (RTCRSSLINT_MAGIC). */
54	uint32_t u32Magic;
55	/** Reference count. */
56	uint32_t volatile cRefs;
57	/** The SSL context. */
58	SSL_CTX *pCtx;
59	} RTCRSSLINT;
60
61	/**
62	* SSL session instance data for OpenSSL.
63	*/
64	typedef struct RTCRSSLSESSIONINT
65	{
66	/** Magic value (RTCRSSLSESSIONINT_MAGIC). */
67	uint32_t u32Magic;
68	/** Reference count. */
69	uint32_t volatile cRefs;
70	/** RTCRSSLSESSION_F_XXX. */
71	uint32_t fFlags;
72
73	/** The SSL instance. */
74	SSL *pSsl;
75	/** The socket BIO instance. */
76	BIO *pBio;
77	} RTCRSSLSESSIONINT;
78
79
80
81	RTDECL(int) RTCrSslCreate(PRTCRSSL phSsl, uint32_t fFlags)
82	{
83	AssertPtr(phSsl);
84	*phSsl = NIL_RTCRSSL;
85	AssertReturn(!fFlags, VERR_INVALID_FLAGS);
86
87	const SSL_METHOD *pSslMethod = TLS_server_method();
88	if (pSslMethod)
89	{
90	RTCRSSLINT pThis = (RTCRSSLINT )RTMemAllocZ(sizeof(*pThis));
91	if (pThis)
92	{
93	pThis->pCtx = SSL_CTX_new(pSslMethod);
94	if (pThis->pCtx)
95	{
96	pThis->u32Magic = RTCRSSLINT_MAGIC;
97	pThis->cRefs = 1;
98
99	*phSsl = pThis;
100	return VINF_SUCCESS;
101	}
102	}
103	return VERR_NO_MEMORY;
104	}
105	return VERR_NOT_SUPPORTED;
106	}
107
108
109	RTDECL(uint32_t) RTCrSslRetain(RTCRSSL hSsl)
110	{
111	RTCRSSLINT *pThis = hSsl;
112	AssertPtrReturn(pThis, UINT32_MAX);
113	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, UINT32_MAX);
114
115	uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
116	Assert(cRefs > 1);
117	Assert(cRefs < 1024);
118	return cRefs;
119	}
120
121
122	/**
123	* Worker for RTCrSslRelease.
124	*/
125	static int rtCrSslDestroy(RTCRSSLINT *pThis)
126	{
127	ASMAtomicWriteU32(&pThis->u32Magic, ~RTCRSSLINT_MAGIC);
128	SSL_CTX_free(pThis->pCtx);
129	pThis->pCtx = NULL;
130	RTMemFree(pThis);
131	return 0;
132	}
133
134
135	RTDECL(uint32_t) RTCrSslRelease(RTCRSSL hSsl)
136	{
137	RTCRSSLINT *pThis = hSsl;
138	if (pThis == NIL_RTCRSSL)
139	return 0;
140	AssertPtrReturn(pThis, UINT32_MAX);
141	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, UINT32_MAX);
142
143	uint32_t cRefs = ASMAtomicDecU32(&pThis->cRefs);
144	Assert(cRefs < 1024);
145	if (cRefs == 0)
146	return rtCrSslDestroy(pThis);
147	return cRefs;
148	}
149
150
151	RTDECL(int) RTCrSslSetCertificateFile(RTCRSSL hSsl, const char *pszFile, uint32_t fFlags)
152	{
153	RTCRSSLINT *pThis = hSsl;
154	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
155	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
156	AssertReturn(!(fFlags & ~RTCRSSL_FILE_F_ASN1), VERR_INVALID_FLAGS);
157
158	int rcOssl = SSL_CTX_use_certificate_file(pThis->pCtx, pszFile,
159	RTCRSSL_FILE_F_ASN1 & fFlags ? SSL_FILETYPE_PEM : SSL_FILETYPE_ASN1);
160	if (rcOssl != 0)
161	return VINF_SUCCESS;
162	return !RTFileExists(pszFile) ? VERR_FILE_NOT_FOUND : VERR_OPEN_FAILED; /** @todo Better status codes */
163	}
164
165
166	RTDECL(int) RTCrSslSetPrivateKeyFile(RTCRSSL hSsl, const char *pszFile, uint32_t fFlags)
167	{
168	RTCRSSLINT *pThis = hSsl;
169	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
170	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
171	AssertReturn(!(fFlags & ~RTCRSSL_FILE_F_ASN1), VERR_INVALID_FLAGS);
172
173	int rcOssl = SSL_CTX_use_PrivateKey_file(pThis->pCtx, pszFile,
174	RTCRSSL_FILE_F_ASN1 & fFlags ? SSL_FILETYPE_PEM : SSL_FILETYPE_ASN1);
175	if (rcOssl != 0)
176	return VINF_SUCCESS;
177	return !RTFileExists(pszFile) ? VERR_FILE_NOT_FOUND : VERR_OPEN_FAILED; /** @todo Better status codes */
178	}
179
180
181	RTDECL(int) RTCrSslLoadTrustedRootCerts(RTCRSSL hSsl, const char pszFile, const char pszDir)
182	{
183	RTCRSSLINT *pThis = hSsl;
184	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
185	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
186
187	int rcOssl = SSL_CTX_load_verify_locations(pThis->pCtx, pszFile, pszDir);
188	if (rcOssl != 0)
189	return VINF_SUCCESS;
190
191	if (pszFile && !RTFileExists(pszFile))
192	return VERR_FILE_NOT_FOUND;
193	return VERR_OPEN_FAILED; /** @todo Better status codes */
194	}
195
196
197	RTDECL(int) RTCrSslSetNoPeerVerify(RTCRSSL hSsl)
198	{
199	RTCRSSLINT *pThis = hSsl;
200	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
201	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
202
203	SSL_CTX_set_verify(pThis->pCtx, SSL_VERIFY_NONE, NULL);
204	return VINF_SUCCESS;
205	}
206
207
208
209	//RTDECL(int) RTCrSslCreateSession(RTCRSSL hSsl, RTSOCKET hSocket, uint32_t fFlags, PRTCRSSLSESSION phSslConn);
210
211	RTDECL(int) RTCrSslCreateSessionForNativeSocket(RTCRSSL hSsl, RTHCINTPTR hNativeSocket, uint32_t fFlags,
212	PRTCRSSLSESSION phSslSession)
213	{
214	/*
215	* Validate input.
216	*/
217	*phSslSession = NIL_RTCRSSLSESSION;
218
219	RTCRSSLINT *pThis = hSsl;
220	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
221	AssertReturn(pThis->u32Magic == RTCRSSLINT_MAGIC, VERR_INVALID_HANDLE);
222	AssertReturn(!(fFlags & ~RTCRSSLSESSION_F_NON_BLOCKING), VERR_INVALID_FLAGS);
223
224	/*
225	* Create a new session.
226	*/
227	int rc = VERR_NO_MEMORY;
228	RTCRSSLSESSIONINT pSession = (RTCRSSLSESSIONINT )RTMemAllocZ(sizeof(*pSession));
229	if (pSession)
230	{
231	pSession->pSsl = SSL_new(pThis->pCtx);
232	if (pSession->pSsl)
233	{
234	/* Disable read-ahead if non-blocking socket relying on select/poll. */
235	if (fFlags & RTCRSSLSESSION_F_NON_BLOCKING)
236	SSL_set_read_ahead(pSession->pSsl, 0);
237
238	/* Create a wrapper for the socket handle. */
239	pSession->pBio = BIO_new_socket(hNativeSocket, BIO_NOCLOSE);
240	if (pSession->pBio)
241	{
242	BIO_up_ref(pSession->pBio); /* our reference. */
243	SSL_set_bio(pSession->pSsl, pSession->pBio, pSession->pBio);
244
245	/*
246	* Done.
247	*/
248	pSession->cRefs = 1;
249	pSession->u32Magic = RTCRSSLSESSIONINT_MAGIC;
250	*phSslSession = pSession;
251	return VINF_SUCCESS;
252	}
253
254	SSL_free(pSession->pSsl);
255	pSession->pSsl = NULL;
256	}
257	RTMemFree(pThis);
258	}
259	return rc;
260	}
261
262
263	/*********************************************************************************************************************************
264	* Session implementation. *
265	*********************************************************************************************************************************/
266
267	RTDECL(uint32_t) RTCrSslSessionRetain(RTCRSSLSESSION hSslSession)
268	{
269	RTCRSSLSESSIONINT *pThis = hSslSession;
270	AssertPtrReturn(pThis, UINT32_MAX);
271	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, UINT32_MAX);
272
273	uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
274	Assert(cRefs > 1);
275	Assert(cRefs < 1024);
276	return cRefs;
277	}
278
279
280	/**
281	* Worker for RTCrSslRelease.
282	*/
283	static int rtCrSslSessionDestroy(RTCRSSLSESSIONINT *pThis)
284	{
285	ASMAtomicWriteU32(&pThis->u32Magic, ~RTCRSSLSESSIONINT_MAGIC);
286	SSL_free(pThis->pSsl);
287	pThis->pSsl = NULL;
288	BIO_free(pThis->pBio);
289	pThis->pBio = NULL;
290	RTMemFree(pThis);
291	return 0;
292	}
293
294
295	RTDECL(uint32_t) RTCrSslSessionRelease(RTCRSSLSESSION hSslSession)
296	{
297	RTCRSSLSESSIONINT *pThis = hSslSession;
298	if (pThis == NIL_RTCRSSLSESSION)
299	return 0;
300	AssertPtrReturn(pThis, UINT32_MAX);
301	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, UINT32_MAX);
302
303	uint32_t cRefs = ASMAtomicDecU32(&pThis->cRefs);
304	Assert(cRefs < 1024);
305	if (cRefs == 0)
306	return rtCrSslSessionDestroy(pThis);
307	return cRefs;
308	}
309
310
311	RTDECL(int) RTCrSslSessionAccept(RTCRSSLSESSION hSslSession, uint32_t fFlags)
312	{
313	RTCRSSLSESSIONINT *pThis = hSslSession;
314	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
315	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
316	AssertReturn(!fFlags, VERR_INVALID_FLAGS);
317
318	int rcOssl = SSL_accept(pThis->pSsl);
319	if (rcOssl > 0)
320	return VINF_SUCCESS;
321
322	/** @todo better status codes. */
323	if (BIO_should_retry(pThis->pBio))
324	return VERR_TRY_AGAIN;
325	return VERR_NOT_SUPPORTED;
326	}
327
328
329	RTDECL(int) RTCrSslSessionConnect(RTCRSSLSESSION hSslSession, uint32_t fFlags)
330	{
331	RTCRSSLSESSIONINT *pThis = hSslSession;
332	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
333	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
334	AssertReturn(!fFlags, VERR_INVALID_FLAGS);
335
336	int rcOssl = SSL_connect(pThis->pSsl);
337	if (rcOssl > 0)
338	return VINF_SUCCESS;
339
340	/** @todo better status codes. */
341	if (BIO_should_retry(pThis->pBio))
342	return VERR_TRY_AGAIN;
343	return VERR_NOT_SUPPORTED;
344	}
345
346
347	RTDECL(const char *) RTCrSslSessionGetVersion(RTCRSSLSESSION hSslSession)
348	{
349	RTCRSSLSESSIONINT *pThis = hSslSession;
350	AssertPtrReturn(pThis, NULL);
351	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, NULL);
352
353	return SSL_get_version(pThis->pSsl);
354	}
355
356
357	RTDECL(int) RTCrSslSessionGetCertIssuerNameAsString(RTCRSSLSESSION hSslSession, char pszBuf, size_t cbBuf, size_t pcbActual)
358	{
359	RTCRSSLSESSIONINT *pThis = hSslSession;
360	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
361	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
362	AssertPtrNull(pszBuf);
363	AssertPtrNull(pcbActual);
364	if (*pcbActual)
365	pcbActual = 0;
366
367	/*
368	* Get and format the certificate issuer name.
369	*/
370	int rc = VERR_NOT_AVAILABLE;
371	X509 *pCert = SSL_get_certificate(pThis->pSsl);
372	if (pCert)
373	{
374	X509_NAME *pIssuer = X509_get_issuer_name(pCert);
375	if (pIssuer)
376	{
377	char *pszSrc = X509_NAME_oneline(pIssuer, NULL, 0);
378	if (pszSrc)
379	{
380	/*
381	* Copy out the result and free it.
382	*/
383	size_t cbNeeded = strlen(pszSrc) + 1;
384	if (*pcbActual)
385	*pcbActual = cbNeeded;
386
387	if (pszBuf != NULL && cbBuf > 0)
388	{
389	if (cbBuf >= cbNeeded)
390	{
391	memcpy(pszBuf, pszSrc, cbNeeded);
392	rc = VINF_SUCCESS;
393	}
394	else
395	{
396	memcpy(pszBuf, pszSrc, cbBuf - 1);
397	pszBuf[cbBuf - 1] = '\0';
398	rc = VERR_BUFFER_OVERFLOW;
399	}
400	}
401	else
402	rc = VERR_BUFFER_OVERFLOW;
403	OPENSSL_free(pszSrc);
404	}
405	}
406	}
407	return rc;
408	}
409
410
411	RTDECL(bool) RTCrSslSessionPending(RTCRSSLSESSION hSslSession)
412	{
413	RTCRSSLSESSIONINT *pThis = hSslSession;
414	AssertPtrReturn(pThis, true);
415	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, true);
416
417	return SSL_pending(pThis->pSsl) != 0;
418	}
419
420
421	RTDECL(ssize_t) RTCrSslSessionRead(RTCRSSLSESSION hSslSession, void *pvBuf, size_t cbToRead)
422	{
423	RTCRSSLSESSIONINT *pThis = hSslSession;
424	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
425	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
426
427	Assert((size_t)(int)cbToRead == cbToRead);
428
429	int cbActual = SSL_read(pThis->pSsl, pvBuf, (int)cbToRead);
430	if (cbActual > 0)
431	return cbActual;
432	if (BIO_should_retry(pThis->pBio))
433	return VERR_TRY_AGAIN;
434	return VERR_READ_ERROR; /** @todo better status codes. */
435	}
436
437
438	RTDECL(ssize_t) RTCrSslSessionWrite(RTCRSSLSESSION hSslSession, void const *pvBuf, size_t cbToWrite)
439	{
440	RTCRSSLSESSIONINT *pThis = hSslSession;
441	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
442	AssertReturn(pThis->u32Magic == RTCRSSLSESSIONINT_MAGIC, VERR_INVALID_HANDLE);
443
444	Assert((size_t)(int)cbToWrite == cbToWrite);
445	Assert(cbToWrite != 0 /* undefined behavior if zero */);
446
447	int cbActual = SSL_write(pThis->pSsl, pvBuf, (int)cbToWrite);
448	if (cbActual > 0)
449	return cbActual;
450	if (BIO_should_retry(pThis->pBio))
451	return VERR_TRY_AGAIN;
452	return VERR_WRITE_ERROR; /** @todo better status codes. */
453	}
454
455	#endif /* IPRT_WITH_OPENSSL */
456

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/crypto/ssl-openssl.cpp@ 74295

Download in other formats: