stack-vcc.asm@ 96407

Last change on this file since 96407 was 96407, checked in by vboxsync, 3 years ago
scm copyright and license note update
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 14.0 KB

Line
1	; $Id: stack-vcc.asm 96407 2022-08-22 17:43:14Z vboxsync $
2	;; @file
3	; IPRT - Stack related Visual C++ support routines.
4	;
5
6	;
7	; Copyright (C) 2022 Oracle and/or its affiliates.
8	;
9	; This file is part of VirtualBox base platform packages, as
10	; available from https://www.215389.xyz.
11	;
12	; This program is free software; you can redistribute it and/or
13	; modify it under the terms of the GNU General Public License
14	; as published by the Free Software Foundation, in version 3 of the
15	; License.
16	;
17	; This program is distributed in the hope that it will be useful, but
18	; WITHOUT ANY WARRANTY; without even the implied warranty of
19	; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	; General Public License for more details.
21	;
22	; You should have received a copy of the GNU General Public License
23	; along with this program; if not, see <https://www.gnu.org/licenses>.
24	;
25	; The contents of this file may alternatively be used under the terms
26	; of the Common Development and Distribution License Version 1.0
27	; (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28	; in the VirtualBox distribution, in which case the provisions of the
29	; CDDL are applicable instead of those of the GPL.
30	;
31	; You may elect to license modified versions of this file under the
32	; terms and conditions of either the GPL or the CDDL or both.
33	;
34	; SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35	;
36
37
38
39	;*********************************************************************************************************************************
40	;* Header Files *
41	;*********************************************************************************************************************************
42	%if 0 ; YASM's builtin SEH64 support doesn't cope well with code alignment, so use our own.
43	%define RT_ASM_WITH_SEH64
44	%else
45	%define RT_ASM_WITH_SEH64_ALT
46	%endif
47	%include "iprt/asmdefs.mac"
48	%include "iprt/x86.mac"
49
50
51	;*********************************************************************************************************************************
52	;* Structures and Typedefs *
53	;*********************************************************************************************************************************
54
55	;; Variable descriptor.
56	struc RTC_VAR_DESC_T
57	.offFrame resd 1
58	.cbVar resd 1
59	alignb RTCCPTR_CB
60	.pszName RTCCPTR_RES 1
61	endstruc
62
63	;; Frame descriptor.
64	struc RTC_FRAME_DESC_T
65	.cVars resd 1
66	alignb RTCCPTR_CB
67	.paVars RTCCPTR_RES 1 ; Array of RTC_VAR_DESC_T.
68	endstruc
69
70	;; An alloca allocation.
71	struc RTC_ALLOCA_ENTRY_T
72	.uGuard1 resd 1
73	.pNext RTCCPTR_RES 1 ; Misaligned.
74	%if ARCH_BITS == 32
75	.pNextPad resd 1
76	%endif
77	.cb RTCCPTR_RES 1 ; Misaligned.
78	%if ARCH_BITS == 32
79	.cbPad resd 1
80	%endif
81	.auGuard2 resd 3
82	endstruc
83
84	%ifdef RT_ARCH_X86
85	%define FASTCALL_NAME(a_Name, a_cbArgs) $@ %+ a_Name %+ @ %+ a_cbArgs
86	%else
87	%define FASTCALL_NAME(a_Name, a_cbArgs) NAME(a_Name)
88	%endif
89
90
91	;*********************************************************************************************************************************
92	;* Defined Constants And Macros *
93	;*********************************************************************************************************************************
94	%define VARIABLE_MARKER_PRE 0xcccccccc
95	%define VARIABLE_MARKER_POST 0xcccccccc
96
97	%define ALLOCA_FILLER_BYTE 0xcc
98	%define ALLOCA_FILLER_32 0xcccccccc
99
100
101	;*********************************************************************************************************************************
102	;* Global Variables *
103	;*********************************************************************************************************************************
104	BEGINDATA
105	GLOBALNAME __security_cookie
106	dd 0xdeadbeef
107	dd 0x0c00ffe0
108
109
110	;*********************************************************************************************************************************
111	;* External Symbols *
112	;*********************************************************************************************************************************
113	BEGINCODE
114	extern NAME(_RTC_StackVarCorrupted)
115	extern NAME(_RTC_SecurityCookieMismatch)
116	%ifdef RT_ARCH_X86
117	extern NAME(_RTC_CheckEspFailed)
118	%endif
119
120
121	BEGINPROC __GSHandlerCheck
122	SEH64_END_PROLOGUE
123	int3
124	ENDPROC __GSHandlerCheck
125
126	;;
127	; Probe stack to trigger guard faults, and for x86 to allocate stack space.
128	;
129	; @param xAX Frame size.
130	; @uses AMD64: Nothing (because we don't quite now the convention).
131	; x86: ESP = ESP - EAX; nothing else
132	;
133	ALIGNCODE(64)
134	GLOBALNAME_RAW __alloca_probe, __alloca_probe, function
135	BEGINPROC_RAW __chkstk
136	push xBP
137	SEH64_PUSH_xBP
138	mov xBP, xSP
139	SEH64_SET_FRAME_xBP 0
140	push xAX
141	SEH64_PUSH_GREG xAX
142	push xBX
143	SEH64_PUSH_GREG xBX
144	SEH64_END_PROLOGUE
145
146	;
147	; Adjust eax so we can use xBP for stack addressing.
148	;
149	sub xAX, xCB*2
150	jle .touch_loop_done
151
152	;
153	; Subtract what's left of the current page from eax and only engage
154	; the touch loop if (int)xAX > 0.
155	;
156	mov ebx, PAGE_SIZE - 1
157	and ebx, ebp
158	sub xAX, xBX
159	jnl .touch_loop
160
161	.touch_loop_done:
162	pop xBX
163	pop xAX
164	leave
165	%ifndef RT_ARCH_X86
166	ret
167	%else
168	;
169	; Do the stack space allocation and jump to the return location.
170	;
171	sub esp, eax
172	add esp, 4
173	jmp dword [esp + eax - 4]
174	%endif
175
176	;
177	; The touch loop.
178	;
179	.touch_loop:
180	sub xBX, PAGE_SIZE
181	mov [xBP + xBX], bl
182	sub xAX, PAGE_SIZE
183	jnl .touch_loop
184	jmp .touch_loop_done
185	ENDPROC_RAW __chkstk
186
187
188	%ifdef RT_ARCH_X86
189	;;
190	; 8 and 16 byte aligned alloca w/ probing.
191	;
192	; This routine adjusts the allocation size so __chkstk will return a
193	; correctly aligned allocation.
194	;
195	; @param xAX Unaligned allocation size.
196	;
197	%macro __alloc_probe_xxx 1
198	ALIGNCODE(16)
199	BEGINPROC_RAW __alloca_probe_ %+ %1
200	push ecx
201
202	;
203	; Calc the ESP address after the allocation and adjust EAX so that it
204	; will be aligned as desired.
205	;
206	lea ecx, [esp + 8]
207	sub ecx, eax
208	and ecx, %1 - 1
209	add eax, ecx
210	jc .bad_alloc_size
211	.continue:
212
213	pop ecx
214	jmp __alloca_probe
215
216	.bad_alloc_size:
217	%ifdef RT_STRICT
218	int3
219	%endif
220	or eax, 0xfffffff0
221	jmp .continue
222	ENDPROC_RAW __alloca_probe_ %+ %1
223	%endmacro
224
225	__alloc_probe_xxx 16
226	__alloc_probe_xxx 8
227	%endif ; RT_ARCH_X86
228
229
230	;;
231	; This just initializes a global and calls _RTC_SetErrorFuncW to NULL, and
232	; since we don't have either of those we have nothing to do here.
233	BEGINPROC _RTC_InitBase
234	SEH64_END_PROLOGUE
235	ret
236	ENDPROC _RTC_InitBase
237
238
239	;;
240	; Nothing to do here.
241	BEGINPROC _RTC_Shutdown
242	SEH64_END_PROLOGUE
243	ret
244	ENDPROC _RTC_Shutdown
245
246
247
248
249	;;
250	; Checks stack variable markers.
251	;
252	; This seems to be a regular C function in the CRT, but x86 is conveniently
253	; using the fastcall convention which makes it very similar to amd64.
254	;
255	; We try make this as sleek as possible, leaving all the trouble for when we
256	; find a corrupted stack variable and need to call a C function to complain.
257	;
258	; @param pStackFrame The caller RSP/ESP. [RCX/ECX]
259	; @param pFrameDesc Frame descriptor. [RDX/EDX]
260	;
261	ALIGNCODE(64)
262	BEGINPROC_RAW FASTCALL_NAME(_RTC_CheckStackVars, 8)
263	push xBP
264	SEH64_PUSH_xBP
265	SEH64_END_PROLOGUE
266
267	;
268	; Load the variable count into eax and check that it's not zero.
269	;
270	mov eax, [xDX + RTC_FRAME_DESC_T.cVars]
271	test eax, eax
272	jz .return
273
274	;
275	; Make edx/rdx point to the current variable and xBP be the frame pointer.
276	; The latter frees up xCX for scratch use and incidentally make stack access
277	; go via SS instead of DS (mostly irrlevant in 64-bit and 32-bit mode).
278	;
279	mov xDX, [xDX + RTC_FRAME_DESC_T.paVars]
280	mov xBP, xCX
281
282	;
283	; Loop thru the variables and check that their markers/fences haven't be
284	; trampled over.
285	;
286	.next_var:
287	; Marker before the variable.
288	%if ARCH_BITS == 64
289	movsxd rcx, dword [xDX + RTC_VAR_DESC_T.offFrame]
290	%else
291	mov xCX, dword [xDX + RTC_VAR_DESC_T.offFrame]
292	%endif
293	cmp dword [xBP + xCX - 4], VARIABLE_MARKER_PRE
294	jne .corrupted
295
296	; Marker after the variable.
297	add ecx, dword [xDX + RTC_VAR_DESC_T.cbVar]
298	%if ARCH_BITS == 64
299	movsxd rcx, ecx
300	%endif
301	cmp dword [xBP + xCX], VARIABLE_MARKER_POST
302	jne .corrupted
303
304	;
305	; Advance to the next variable.
306	;
307	.advance:
308	add xDX, RTC_VAR_DESC_T_size
309	dec eax
310	jnz .next_var
311
312	;
313	; Return.
314	;
315	.return:
316	pop xBP
317	ret
318
319	;
320	; Complain about corrupt variable.
321	;
322	.corrupted:
323	push xAX
324	push xDX
325	%ifdef RT_ARCH_AMD64
326	sub xSP, 28h
327	mov xCX, xBP ; frame pointer + variable descriptor.
328	%else
329	push xBP ; save EBP
330	push xDX ; parameter 2 - variable descriptor
331	push xBP ; parameter 1 - frame pointer.
332	lea xBP, [xSP + 3*xCB] ; turn it into a frame pointer during the call for better unwind.
333	%endif
334
335	call NAME(_RTC_StackVarCorrupted)
336
337	%ifdef RT_ARCH_AMD64
338	add xSP, 28h
339	%else
340	add xSP, xCB * 4 ; parameters
341	pop xBP
342	%endif
343	pop xDX
344	pop xAX
345	jmp .advance
346	ENDPROC_RAW FASTCALL_NAME(_RTC_CheckStackVars, 8)
347
348
349	%ifdef RT_ARCH_X86
350	;;
351	; Called to follow up on a 'CMP ESP, EBP' kind of instruction,
352	; expected to report failure if the compare failed.
353	;
354	ALIGNCODE(16)
355	BEGINPROC _RTC_CheckEsp
356	jne .unexpected_esp
357	ret
358
359	.unexpected_esp:
360	push ebp
361	mov ebp, esp
362	push eax
363	push ecx
364	push edx
365
366	; DECLASM(void) _RTC_CheckEspFailed(uintptr_t uEip, uintptr_t uEsp, uintptr_t uEbp)
367	push dword [ebp]
368	lea edx, [ebp + 8]
369	push edx
370	mov ecx, [ebp + 8]
371	push ecx
372	call NAME(_RTC_CheckEspFailed)
373
374	pop edx
375	pop ecx
376	pop eax
377	leave
378	ret
379	ENDPROC _RTC_CheckEsp
380	%endif ; RT_ARCH_X86
381
382
383
384	;;
385	; Initialize an alloca allocation list entry and add it to it.
386	;
387	; When this is call, presumably _RTC_CheckStackVars2 is used to verify the frame.
388	;
389	; @param pNewEntry Pointer to the new entry. [RCX/ECX]
390	; @param cbEntry The entry size, including header. [RDX/EDX]
391	; @param ppHead Pointer to the list head pointer. [R8/stack]
392	;
393	ALIGNCODE(64)
394	BEGINPROC_RAW FASTCALL_NAME(_RTC_AllocaHelper, 12)
395	SEH64_END_PROLOGUE
396
397	;
398	; Check that input isn't NULL or the size isn't zero.
399	;
400	test xCX, xCX
401	jz .return
402	test xDX, xDX
403	jz .return
404	%if ARCH_BITS == 64
405	test r8, r8
406	%else
407	cmp dword [xSP + xCB], 0
408	%endif
409	jz .return
410
411	;
412	; Memset the memory to ALLOCA_FILLER
413	;
414	%if ARCH_BITS == 64
415	mov r10, rdi ; save rdi
416	mov r11, rcx ; save pNewEntry
417	%else
418	push xDI
419	push xCX
420	cld ; paranoia
421	%endif
422
423	mov al, ALLOCA_FILLER_BYTE
424	mov xDI, xCX ; entry pointer
425	mov xCX, xDX ; entry size (in bytes)
426	rep stosb
427
428	%if ARCH_BITS == 64
429	mov rdi, r10
430	%else
431	pop xCX
432	pop xDI
433	%endif
434
435	;
436	; Fill in the entry and link it as onto the head of the chain.
437	;
438	%if ARCH_BITS == 64
439	mov [r11 + RTC_ALLOCA_ENTRY_T.cb], xDX
440	mov xAX, [r8]
441	mov [r11 + RTC_ALLOCA_ENTRY_T.pNext], xAX
442	mov [r8], r11
443	%else
444	mov [xCX + RTC_ALLOCA_ENTRY_T.cb], xDX
445	mov xAX, [xSP + xCB] ; ppHead
446	mov xDX, [xAX]
447	mov [xCX + RTC_ALLOCA_ENTRY_T.pNext], xDX
448	mov [xAX], xCX
449	%endif
450
451	.return:
452	%if ARCH_BITS == 64
453	ret
454	%else
455	ret 4
456	%endif
457	ENDPROC_RAW FASTCALL_NAME(_RTC_AllocaHelper, 12)
458
459
460	;;
461	; Checks if the secuity cookie ok, complaining and terminating if it isn't.
462	;
463	ALIGNCODE(16)
464	BEGINPROC_RAW FASTCALL_NAME(__security_check_cookie, 4)
465	SEH64_END_PROLOGUE
466	cmp xCX, [NAME(__security_cookie) xWrtRIP]
467	jne .corrupted
468	;; amd64 version checks if the top 16 bits are zero, we skip that for now.
469	ret
470
471	.corrupted:
472	%ifdef RT_ARCH_AMD64
473	jmp NAME(_RTC_SecurityCookieMismatch)
474	%else
475	push ebp
476	mov ebp, esp
477	push ecx
478	call NAME(_RTC_SecurityCookieMismatch)
479	pop ecx
480	leave
481	ret
482	%endif
483	ENDPROC_RAW FASTCALL_NAME(__security_check_cookie, 4)
484
485
486
487	; Not stack related stubs.
488	BEGINPROC __C_specific_handler
489	SEH64_END_PROLOGUE
490	int3
491	ENDPROC __C_specific_handler
492
493
494	BEGINPROC __report_rangecheckfailure
495	SEH64_END_PROLOGUE
496	int3
497	ENDPROC __report_rangecheckfailure
498

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/compiler/vcc/stack-vcc.asm@ 96407

Download in other formats: