VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/compiler/vcc/stack-vcc.asm@ 96370

Last change on this file since 96370 was 96370, checked in by vboxsync, 3 years ago

IPRT/nocrt: x86 stack support routines. bugref:10261
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 13.7 KB
Line 
1; $Id: stack-vcc.asm 96370 2022-08-20 02:28:31Z vboxsync $
2;; @file
3; IPRT - Stack related Visual C++ support routines.
4;
5
6;
7; Copyright (C) 2022 Oracle Corporation
8;
9; This file is part of VirtualBox Open Source Edition (OSE), as
10; available from http://www.215389.xyz. This file is free software;
11; you can redistribute it and/or modify it under the terms of the GNU
12; General Public License (GPL) as published by the Free Software
13; Foundation, in version 2 as it comes in the "COPYING" file of the
14; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16;
17; The contents of this file may alternatively be used under the terms
18; of the Common Development and Distribution License Version 1.0
19; (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20; VirtualBox OSE distribution, in which case the provisions of the
21; CDDL are applicable instead of those of the GPL.
22;
23; You may elect to license modified versions of this file under the
24; terms and conditions of either the GPL or the CDDL or both.
25;
26
27
28
29;*********************************************************************************************************************************
30;* Header Files *
31;*********************************************************************************************************************************
32%if 0 ; YASM's builtin SEH64 support doesn't cope well with code alignment, so use our own.
33 %define RT_ASM_WITH_SEH64
34%else
35 %define RT_ASM_WITH_SEH64_ALT
36%endif
37%include "iprt/asmdefs.mac"
38%include "iprt/x86.mac"
39
40
41;*********************************************************************************************************************************
42;* Structures and Typedefs *
43;*********************************************************************************************************************************
44
45;; Variable descriptor.
46struc RTC_VAR_DESC_T
47 .offFrame resd 1
48 .cbVar resd 1
49 alignb RTCCPTR_CB
50 .pszName RTCCPTR_RES 1
51endstruc
52
53;; Frame descriptor.
54struc RTC_FRAME_DESC_T
55 .cVars resd 1
56 alignb RTCCPTR_CB
57 .paVars RTCCPTR_RES 1 ; Array of RTC_VAR_DESC_T.
58endstruc
59
60;; An alloca allocation.
61struc RTC_ALLOCA_ENTRY_T
62 .uGuard1 resd 1
63 .pNext RTCCPTR_RES 1 ; Misaligned.
64%if ARCH_BITS == 32
65 .pNextPad resd 1
66%endif
67 .cb RTCCPTR_RES 1 ; Misaligned.
68%if ARCH_BITS == 32
69 .cbPad resd 1
70%endif
71 .auGuard2 resd 3
72endstruc
73
74%ifdef RT_ARCH_X86
75 %define FASTCALL_NAME(a_Name, a_cbArgs) $@ %+ a_Name %+ @ %+ a_cbArgs
76%else
77 %define FASTCALL_NAME(a_Name, a_cbArgs) NAME(a_Name)
78%endif
79
80
81;*********************************************************************************************************************************
82;* Defined Constants And Macros *
83;*********************************************************************************************************************************
84%define VARIABLE_MARKER_PRE 0xcccccccc
85%define VARIABLE_MARKER_POST 0xcccccccc
86
87%define ALLOCA_FILLER_BYTE 0xcc
88%define ALLOCA_FILLER_32 0xcccccccc
89
90
91;*********************************************************************************************************************************
92;* Global Variables *
93;*********************************************************************************************************************************
94BEGINDATA
95GLOBALNAME __security_cookie
96 dd 0xdeadbeef
97 dd 0x0c00ffe0
98
99
100;*********************************************************************************************************************************
101;* External Symbols *
102;*********************************************************************************************************************************
103BEGINCODE
104extern NAME(_RTC_StackVarCorrupted)
105extern NAME(_RTC_SecurityCookieMismatch)
106%ifdef RT_ARCH_X86
107extern NAME(_RTC_CheckEspFailed)
108%endif
109
110
111BEGINPROC __GSHandlerCheck
112 SEH64_END_PROLOGUE
113 int3
114ENDPROC __GSHandlerCheck
115
116;;
117; Probe stack to trigger guard faults, and for x86 to allocate stack space.
118;
119; @param xAX Frame size.
120; @uses AMD64: Nothing (because we don't quite now the convention).
121; x86: ESP = ESP - EAX; nothing else
122;
123ALIGNCODE(64)
124GLOBALNAME_RAW __alloca_probe, __alloca_probe, function
125BEGINPROC_RAW __chkstk
126 push xBP
127 SEH64_PUSH_xBP
128 mov xBP, xSP
129 SEH64_SET_FRAME_xBP 0
130 push xAX
131 SEH64_PUSH_GREG xAX
132 push xBX
133 SEH64_PUSH_GREG xBX
134 SEH64_END_PROLOGUE
135
136 ;
137 ; Adjust eax so we can use xBP for stack addressing.
138 ;
139 sub xAX, xCB*2
140 jle .touch_loop_done
141
142 ;
143 ; Subtract what's left of the current page from eax and only engage
144 ; the touch loop if (int)xAX > 0.
145 ;
146 mov ebx, PAGE_SIZE - 1
147 and ebx, ebp
148 sub xAX, xBX
149 jnl .touch_loop
150
151.touch_loop_done:
152 pop xBX
153 pop xAX
154 leave
155%ifndef RT_ARCH_X86
156 ret
157%else
158 ;
159 ; Do the stack space allocation and jump to the return location.
160 ;
161 sub esp, eax
162 add esp, 4
163 jmp dword [esp + eax - 4]
164%endif
165
166 ;
167 ; The touch loop.
168 ;
169.touch_loop:
170 sub xBX, PAGE_SIZE
171 mov [xBP + xBX], bl
172 sub xAX, PAGE_SIZE
173 jnl .touch_loop
174 jmp .touch_loop_done
175ENDPROC_RAW __chkstk
176
177
178%ifdef RT_ARCH_X86
179;;
180; 8 and 16 byte aligned alloca w/ probing.
181;
182; This routine adjusts the allocation size so __chkstk will return a
183; correctly aligned allocation.
184;
185; @param xAX Unaligned allocation size.
186;
187%macro __alloc_probe_xxx 1
188ALIGNCODE(16)
189BEGINPROC_RAW __alloca_probe_ %+ %1
190 push ecx
191
192 ;
193 ; Calc the ESP address after the allocation and adjust EAX so that it
194 ; will be aligned as desired.
195 ;
196 lea ecx, [esp + 8]
197 sub ecx, eax
198 and ecx, %1 - 1
199 add eax, ecx
200 jc .bad_alloc_size
201.continue:
202
203 pop ecx
204 jmp __alloca_probe
205
206.bad_alloc_size:
207 %ifdef RT_STRICT
208 int3
209 %endif
210 or eax, 0xfffffff0
211 jmp .continue
212ENDPROC_RAW __alloca_probe_ %+ %1
213%endmacro
214
215__alloc_probe_xxx 16
216__alloc_probe_xxx 8
217%endif ; RT_ARCH_X86
218
219
220;;
221; This just initializes a global and calls _RTC_SetErrorFuncW to NULL, and
222; since we don't have either of those we have nothing to do here.
223BEGINPROC _RTC_InitBase
224 SEH64_END_PROLOGUE
225 ret
226ENDPROC _RTC_InitBase
227
228
229;;
230; Nothing to do here.
231BEGINPROC _RTC_Shutdown
232 SEH64_END_PROLOGUE
233 ret
234ENDPROC _RTC_Shutdown
235
236
237
238
239;;
240; Checks stack variable markers.
241;
242; This seems to be a regular C function in the CRT, but x86 is conveniently
243; using the fastcall convention which makes it very similar to amd64.
244;
245; We try make this as sleek as possible, leaving all the trouble for when we
246; find a corrupted stack variable and need to call a C function to complain.
247;
248; @param pStackFrame The caller RSP/ESP. [RCX/ECX]
249; @param pFrameDesc Frame descriptor. [RDX/EDX]
250;
251ALIGNCODE(64)
252BEGINPROC_RAW FASTCALL_NAME(_RTC_CheckStackVars, 8)
253 push xBP
254 SEH64_PUSH_xBP
255 SEH64_END_PROLOGUE
256
257 ;
258 ; Load the variable count into eax and check that it's not zero.
259 ;
260 mov eax, [xDX + RTC_FRAME_DESC_T.cVars]
261 test eax, eax
262 jz .return
263
264 ;
265 ; Make edx/rdx point to the current variable and xBP be the frame pointer.
266 ; The latter frees up xCX for scratch use and incidentally make stack access
267 ; go via SS instead of DS (mostly irrlevant in 64-bit and 32-bit mode).
268 ;
269 mov xDX, [xDX + RTC_FRAME_DESC_T.paVars]
270 mov xBP, xCX
271
272 ;
273 ; Loop thru the variables and check that their markers/fences haven't be
274 ; trampled over.
275 ;
276.next_var:
277 ; Marker before the variable.
278%if ARCH_BITS == 64
279 movsxd rcx, dword [xDX + RTC_VAR_DESC_T.offFrame]
280%else
281 mov xCX, dword [xDX + RTC_VAR_DESC_T.offFrame]
282%endif
283 cmp dword [xBP + xCX - 4], VARIABLE_MARKER_PRE
284 jne .corrupted
285
286 ; Marker after the variable.
287 add ecx, dword [xDX + RTC_VAR_DESC_T.cbVar]
288%if ARCH_BITS == 64
289 movsxd rcx, ecx
290%endif
291 cmp dword [xBP + xCX], VARIABLE_MARKER_POST
292 jne .corrupted
293
294 ;
295 ; Advance to the next variable.
296 ;
297.advance:
298 add xDX, RTC_VAR_DESC_T_size
299 dec eax
300 jnz .next_var
301
302 ;
303 ; Return.
304 ;
305.return:
306 pop xBP
307 ret
308
309 ;
310 ; Complain about corrupt variable.
311 ;
312.corrupted:
313 push xAX
314 push xDX
315%ifdef RT_ARCH_AMD64
316 sub xSP, 28h
317 mov xCX, xBP ; frame pointer + variable descriptor.
318%else
319 push xBP ; save EBP
320 push xDX ; parameter 2 - variable descriptor
321 push xBP ; parameter 1 - frame pointer.
322 lea xBP, [xSP + 3*xCB] ; turn it into a frame pointer during the call for better unwind.
323%endif
324
325 call NAME(_RTC_StackVarCorrupted)
326
327%ifdef RT_ARCH_AMD64
328 add xSP, 28h
329%else
330 add xSP, xCB * 4 ; parameters
331 pop xBP
332%endif
333 pop xDX
334 pop xAX
335 jmp .advance
336ENDPROC_RAW FASTCALL_NAME(_RTC_CheckStackVars, 8)
337
338
339%ifdef RT_ARCH_X86
340;;
341; Called to follow up on a 'CMP ESP, EBP' kind of instruction,
342; expected to report failure if the compare failed.
343;
344ALIGNCODE(16)
345BEGINPROC _RTC_CheckEsp
346 jne .unexpected_esp
347 ret
348
349.unexpected_esp:
350 push ebp
351 mov ebp, esp
352 push eax
353 push ecx
354 push edx
355
356 ; DECLASM(void) _RTC_CheckEspFailed(uintptr_t uEip, uintptr_t uEsp, uintptr_t uEbp)
357 push dword [ebp]
358 lea edx, [ebp + 8]
359 push edx
360 mov ecx, [ebp + 8]
361 push ecx
362 call NAME(_RTC_CheckEspFailed)
363
364 pop edx
365 pop ecx
366 pop eax
367 leave
368 ret
369ENDPROC _RTC_CheckEsp
370%endif ; RT_ARCH_X86
371
372
373
374;;
375; Initialize an alloca allocation list entry and add it to it.
376;
377; When this is call, presumably _RTC_CheckStackVars2 is used to verify the frame.
378;
379; @param pNewEntry Pointer to the new entry. [RCX/ECX]
380; @param cbEntry The entry size, including header. [RDX/EDX]
381; @param ppHead Pointer to the list head pointer. [R8/stack]
382;
383ALIGNCODE(64)
384BEGINPROC_RAW FASTCALL_NAME(_RTC_AllocaHelper, 12)
385 SEH64_END_PROLOGUE
386
387 ;
388 ; Check that input isn't NULL or the size isn't zero.
389 ;
390 test xCX, xCX
391 jz .return
392 test xDX, xDX
393 jz .return
394%if ARCH_BITS == 64
395 test r8, r8
396%else
397 cmp dword [xSP + xCB], 0
398%endif
399 jz .return
400
401 ;
402 ; Memset the memory to ALLOCA_FILLER
403 ;
404%if ARCH_BITS == 64
405 mov r10, rdi ; save rdi
406 mov r11, rcx ; save pNewEntry
407%else
408 push xDI
409 push xCX
410 cld ; paranoia
411%endif
412
413 mov al, ALLOCA_FILLER_BYTE
414 mov xDI, xCX ; entry pointer
415 mov xCX, xDX ; entry size (in bytes)
416 rep stosb
417
418%if ARCH_BITS == 64
419 mov rdi, r10
420%else
421 pop xCX
422 pop xDI
423%endif
424
425 ;
426 ; Fill in the entry and link it as onto the head of the chain.
427 ;
428%if ARCH_BITS == 64
429 mov [r11 + RTC_ALLOCA_ENTRY_T.cb], xDX
430 mov xAX, [r8]
431 mov [r11 + RTC_ALLOCA_ENTRY_T.pNext], xAX
432 mov [r8], r11
433%else
434 mov [xCX + RTC_ALLOCA_ENTRY_T.cb], xDX
435 mov xAX, [xSP + xCB] ; ppHead
436 mov xDX, [xAX]
437 mov [xCX + RTC_ALLOCA_ENTRY_T.pNext], xDX
438 mov [xAX], xCX
439%endif
440
441.return:
442%if ARCH_BITS == 64
443 ret
444%else
445 ret 4
446%endif
447ENDPROC_RAW FASTCALL_NAME(_RTC_AllocaHelper, 12)
448
449
450;;
451; Checks if the secuity cookie ok, complaining and terminating if it isn't.
452;
453ALIGNCODE(16)
454BEGINPROC_RAW FASTCALL_NAME(__security_check_cookie, 4)
455 SEH64_END_PROLOGUE
456 cmp xCX, [NAME(__security_cookie) xWrtRIP]
457 jne .corrupted
458 ;; amd64 version checks if the top 16 bits are zero, we skip that for now.
459 ret
460
461.corrupted:
462%ifdef RT_ARCH_AMD64
463 jmp NAME(_RTC_SecurityCookieMismatch)
464%else
465 push ebp
466 mov ebp, esp
467 push ecx
468 call NAME(_RTC_SecurityCookieMismatch)
469 pop ecx
470 leave
471 ret
472%endif
473ENDPROC_RAW FASTCALL_NAME(__security_check_cookie, 4)
474
475
476
477; Not stack related stubs.
478BEGINPROC __C_specific_handler
479 SEH64_END_PROLOGUE
480 int3
481ENDPROC __C_specific_handler
482
483
484BEGINPROC __report_rangecheckfailure
485 SEH64_END_PROLOGUE
486 int3
487ENDPROC __report_rangecheckfailure
488
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette