VirtualBox

source: vbox/trunk/src/VBox/VMM/testcase/tstPDMAsyncCompletionStress.cpp@ 23677

Last change on this file since 23677 was 23677, checked in by vboxsync, 16 years ago

AsyncCompletion: Stress test bugfixing. Still now working yet
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 19.2 KB
Line 
1/* $Id: tstPDMAsyncCompletionStress.cpp 23677 2009-10-10 23:08:11Z vboxsync $ */
2/** @file
3 * PDM Asynchronous Completion Stresstest.
4 *
5 * This testcase is for stress testing the async completion interface.
6 */
7
8/*
9 * Copyright (C) 2008-2009 Sun Microsystems, Inc.
10 *
11 * This file is part of VirtualBox Open Source Edition (OSE), as
12 * available from http://www.215389.xyz. This file is free software;
13 * you can redistribute it and/or modify it under the terms of the GNU
14 * General Public License (GPL) as published by the Free Software
15 * Foundation, in version 2 as it comes in the "COPYING" file of the
16 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
17 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
18 *
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
20 * Clara, CA 95054 USA or visit http://www.sun.com if you need
21 * additional information or have any questions.
22 */
23
24/*******************************************************************************
25* Header Files *
26*******************************************************************************/
27#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
28
29#include "../VMInternal.h" /* UVM */
30#include <VBox/vm.h>
31#include <VBox/uvm.h>
32#include <VBox/pdmasynccompletion.h>
33#include <VBox/vmm.h>
34#include <VBox/cpum.h>
35#include <VBox/err.h>
36#include <VBox/log.h>
37#include <VBox/pdmapi.h>
38#include <VBox/pdmthread.h>
39#include <iprt/alloc.h>
40#include <iprt/asm.h>
41#include <iprt/assert.h>
42#include <iprt/file.h>
43#include <iprt/initterm.h>
44#include <iprt/semaphore.h>
45#include <iprt/rand.h>
46#include <iprt/string.h>
47#include <iprt/path.h>
48#include <iprt/stream.h>
49#include <iprt/thread.h>
50#include <iprt/param.h>
51
52#define TESTCASE "tstPDMAsyncCompletionStress"
53
54#if 0
55/** Number of simultaneous open endpoints for reading and writing. */
56#define NR_OPEN_ENDPOINTS 10
57/** Test pattern size. */
58#define TEST_PATTERN_SIZE (100*_1M)
59/** Minimum file size. */
60#define FILE_SIZE_MIN (100 * _1M)
61/** Maximum file size. */
62#define FILE_SIZE_MAX (10000UL * _1M)
63/** Minimum segment size. */
64#define SEGMENT_SIZE_MIN (512)
65/** Maximum segment size. */
66#define SEGMENT_SIZE_MAX (TEST_PATTERN_SIZE)
67/** Maximum number of active tasks. */
68#define TASK_ACTIVE_MAX (1024)
69/** Maximum size of a transfer. */
70#define TASK_TRANSFER_SIZE_MAX (10*_1M)
71#else
72/** Number of simultaneous open endpoints for reading and writing. */
73#define NR_OPEN_ENDPOINTS 5
74/** Test pattern size. */
75#define TEST_PATTERN_SIZE (10*_1M)
76/** Minimum file size. */
77#define FILE_SIZE_MIN (100 * _1M)
78/** Maximum file size. */
79#define FILE_SIZE_MAX (1000UL * _1M)
80/** Minimum segment size. */
81#define SEGMENT_SIZE_MIN (512)
82/** Maximum segment size. */
83#define SEGMENT_SIZE_MAX (TEST_PATTERN_SIZE)
84/** Maximum number of active tasks. */
85#define TASK_ACTIVE_MAX (512)
86/** Maximum size of a transfer. */
87#define TASK_TRANSFER_SIZE_MAX (_1M)
88#endif
89
90/**
91 * Structure defining a file segment.
92 */
93typedef struct PDMACTESTFILESEG
94{
95 /** Start offset in the file. */
96 RTFOFF off;
97 /** Size of the segment. */
98 size_t cbSegment;
99 /** Pointer to the start of the data in the test pattern used for the segment. */
100 uint8_t *pbData;
101} PDMACTESTFILESEG, *PPDMACTESTFILESEG;
102
103/**
104 * Structure defining a I/O task.
105 */
106typedef struct PDMACTESTFILETASK
107{
108 /** Flag whether the task is currently active. */
109 bool fActive;
110 /** Flag whether this is a write. */
111 bool fWrite;
112 /** Start offset. */
113 RTFOFF off;
114 /** Data segment */
115 PDMDATASEG DataSeg;
116 /** Task handle. */
117 PPDMASYNCCOMPLETIONTASK hTask;
118} PDMACTESTFILETASK, *PPDMACTESTFILETASK;
119
120/**
121 * Structure defining a test file.
122 */
123typedef struct PDMACTESTFILE
124{
125 /** The PDM async completion endpoint handle. */
126 PPDMASYNCCOMPLETIONENDPOINT hEndpoint;
127 /** Template used for this file. */
128 PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
129 /** Maximum size of the file. */
130 uint64_t cbFileMax;
131 /** Current size of the file. */
132 uint64_t cbFileCurr;
133 /** Size of a file segment. */
134 size_t cbFileSegment;
135 /** Maximum number of segments. */
136 unsigned cSegments;
137 /** Pointer to the array describing how the file is assembled
138 * of the test pattern. Used for comparing read data to ensure
139 * that no corruption occured.
140 */
141 PPDMACTESTFILESEG paSegs;
142 /** Maximum number of active tasks for this endpoint. */
143 uint32_t cTasksActiveMax;
144 /** Number of current active tasks. */
145 volatile uint32_t cTasksActiveCurr;
146 /** Pointer to the array of task. */
147 PPDMACTESTFILETASK paTasks;
148 /** I/O thread handle. */
149 PPDMTHREAD hThread;
150 /** Flag whether the thread should terminate. */
151 bool fRunning;
152} PDMACTESTFILE, *PPDMACTESTFILE;
153
154/** Buffer storing the random test pattern. */
155uint8_t *g_pbTestPattern = NULL;
156/** Size of the test pattern. */
157size_t g_cbTestPattern;
158/** Array holding test files. */
159PDMACTESTFILE g_aTestFiles[NR_OPEN_ENDPOINTS];
160
161static void tstPDMACStressTestFileVerify(PPDMACTESTFILE pTestFile, PPDMACTESTFILETASK pTestTask)
162{
163 size_t cbLeft = pTestTask->DataSeg.cbSeg;
164 RTFOFF off = pTestTask->off;
165 uint8_t *pbBuf = (uint8_t *)pTestTask->DataSeg.pvSeg;
166
167 while (cbLeft)
168 {
169 size_t cbCompare;
170 unsigned iSeg = off / pTestFile->cbFileSegment;;
171 PPDMACTESTFILESEG pSeg = &pTestFile->paSegs[iSeg];
172 uint8_t *pbTestPattern;
173 unsigned offSeg = off - pSeg->off;
174
175 cbCompare = RT_MIN(cbLeft, pSeg->cbSegment - offSeg);
176 pbTestPattern = pSeg->pbData + offSeg;
177
178 if (memcmp(pbBuf, pbTestPattern, cbCompare))
179 AssertMsgFailed(("Unexpected data for off=%RTfoff size=%u\n", pTestTask->off, pTestTask->DataSeg.cbSeg));
180
181 pbBuf += cbCompare;
182 off += cbCompare;
183 cbLeft -= cbCompare;
184 }
185}
186
187static void tstPDMACStressTestFileFillBuffer(PPDMACTESTFILE pTestFile, PPDMACTESTFILETASK pTestTask)
188{
189 uint8_t *pbBuf = (uint8_t *)pTestTask->DataSeg.pvSeg;
190 size_t cbLeft = pTestTask->DataSeg.cbSeg;
191 RTFOFF off = pTestTask->off;
192
193 Assert(pTestTask->fWrite && pTestTask->fActive);
194
195 while (cbLeft)
196 {
197 size_t cbFill;
198 unsigned iSeg = off / pTestFile->cbFileSegment;;
199 PPDMACTESTFILESEG pSeg = &pTestFile->paSegs[iSeg];
200 uint8_t *pbTestPattern;
201 unsigned offSeg = off - pSeg->off;
202
203 cbFill = RT_MIN(cbLeft, pSeg->cbSegment - offSeg);
204 pbTestPattern = pSeg->pbData + offSeg;
205
206 memcpy(pbBuf, pbTestPattern, cbFill);
207
208 pbBuf += cbFill;
209 off += cbFill;
210 cbLeft -= cbFill;
211 }
212}
213
214static int tstPDMACStressTestFileWrite(PPDMACTESTFILE pTestFile, PPDMACTESTFILETASK pTestTask)
215{
216 int rc = VINF_SUCCESS;
217
218 Assert(!pTestTask->fActive);
219
220 pTestTask->fActive = true;
221 pTestTask->fWrite = true;
222 pTestTask->DataSeg.cbSeg = RTRandU32Ex(512, TASK_TRANSFER_SIZE_MAX) & ~511;
223
224 RTFOFF offMax;
225
226 /* Did we reached the maximum file size */
227 if (pTestFile->cbFileCurr < pTestFile->cbFileMax)
228 {
229 offMax = (pTestFile->cbFileMax - pTestFile->cbFileCurr) < pTestTask->DataSeg.cbSeg
230 ? pTestFile->cbFileCurr + pTestTask->DataSeg.cbSeg
231 : pTestFile->cbFileCurr;
232 }
233 else
234 offMax = pTestFile->cbFileMax - pTestTask->DataSeg.cbSeg;
235
236 pTestTask->off = RTRandU64Ex(0, offMax) & ~511;
237
238 /* Set new file size of required */
239 if ((uint64_t)pTestTask->off + pTestTask->DataSeg.cbSeg > pTestFile->cbFileCurr)
240 pTestFile->cbFileCurr = pTestTask->off + pTestTask->DataSeg.cbSeg;
241
242 /* Allocate data buffer. */
243 pTestTask->DataSeg.pvSeg = RTMemAlloc(pTestTask->DataSeg.cbSeg);
244 if (!pTestTask->DataSeg.pvSeg)
245 return VERR_NO_MEMORY;
246
247 /* Fill data into buffer. */
248 tstPDMACStressTestFileFillBuffer(pTestFile, pTestTask);
249
250 /* Engage */
251 rc = PDMR3AsyncCompletionEpWrite(pTestFile->hEndpoint, pTestTask->off,
252 &pTestTask->DataSeg, 1,
253 pTestTask->DataSeg.cbSeg,
254 pTestTask,
255 &pTestTask->hTask);
256
257 return rc;
258}
259
260static int tstPDMACStressTestFileRead(PPDMACTESTFILE pTestFile, PPDMACTESTFILETASK pTestTask)
261{
262 int rc = VINF_SUCCESS;
263
264 Assert(!pTestTask->fActive);
265
266 pTestTask->fActive = true;
267 pTestTask->fWrite = false;
268 pTestTask->DataSeg.cbSeg = RTRandU32Ex(1, RT_MIN(pTestFile->cbFileCurr, TASK_TRANSFER_SIZE_MAX));
269
270 AssertMsg(pTestFile->cbFileCurr >= pTestTask->DataSeg.cbSeg, ("Impossible\n"));
271 pTestTask->off = RTRandU64Ex(0, pTestFile->cbFileCurr - pTestTask->DataSeg.cbSeg);
272
273 /* Allocate data buffer. */
274 pTestTask->DataSeg.pvSeg = RTMemAlloc(pTestTask->DataSeg.cbSeg);
275 if (pTestTask->DataSeg.pvSeg)
276 return VERR_NO_MEMORY;
277
278 /* Engage */
279 rc = PDMR3AsyncCompletionEpRead(pTestFile->hEndpoint, pTestTask->off,
280 &pTestTask->DataSeg, 1,
281 pTestTask->DataSeg.cbSeg,
282 pTestTask,
283 &pTestTask->hTask);
284
285 return rc;
286}
287
288/**
289 * Returns true with the given chance in percent.
290 *
291 * @returns true or false
292 * @param iPercentage The percentage of the chance to return true.
293 */
294static bool tstPDMACTestIsTrue(int iPercentage)
295{
296 int uRnd = RTRandU32Ex(0, 100);
297
298 return (uRnd < iPercentage); /* This should be enough for our purpose */
299}
300
301static int tstPDMACTestFileThread(PVM pVM, PPDMTHREAD pThread)
302{
303 PPDMACTESTFILE pTestFile = (PPDMACTESTFILE)pThread->pvUser;
304 int iWriteChance = 100; /* Chance to get a write task in percent. */
305 uint32_t cTasksStarted = 0;
306 int rc = VINF_SUCCESS;
307
308 while (pTestFile->fRunning)
309 {
310 unsigned iTaskCurr = 0;
311
312
313 /* Fill all tasks */
314 while ( (pTestFile->cTasksActiveCurr < pTestFile->cTasksActiveMax)
315 && (iTaskCurr < pTestFile->cTasksActiveMax))
316 {
317 PPDMACTESTFILETASK pTask = &pTestFile->paTasks[iTaskCurr];
318
319 if (!pTask->fActive)
320 {
321 /* Read or write task? */
322 bool fWrite = tstPDMACTestIsTrue(iWriteChance);
323
324 ASMAtomicIncU32(&pTestFile->cTasksActiveCurr);
325
326 if (fWrite)
327 rc = tstPDMACStressTestFileWrite(pTestFile, pTask);
328 else
329 rc = tstPDMACStressTestFileRead(pTestFile, pTask);
330
331 AssertRC(rc);
332
333 cTasksStarted++;
334 }
335
336 iTaskCurr++;
337 }
338
339 /*
340 * Recalc write chance. The bigger the file the lower the chance to have a write.
341 * The minimum chance is 33 percent.
342 */
343 iWriteChance = (int)(((float)100.0 / pTestFile->cbFileMax) * (float)pTestFile->cbFileCurr);
344 iWriteChance = RT_MAX(33, iWriteChance);
345
346 /* Wait a random amount of time. (1ms - 10s) */
347 RTThreadSleep(RTRandU32Ex(1, 10000));
348 }
349
350 /* Wait for the rest to complete. */
351 while (pTestFile->cTasksActiveCurr)
352 RTThreadSleep(250);
353
354 RTPrintf("Thread exiting: processed %u tasks\n", cTasksStarted);
355 return rc;
356}
357
358static void tstPDMACStressTestFileTaskCompleted(PVM pVM, void *pvUser, void *pvUser2)
359{
360 PPDMACTESTFILE pTestFile = (PPDMACTESTFILE)pvUser2;
361 PPDMACTESTFILETASK pTestTask = (PPDMACTESTFILETASK)pvUser;
362
363 if (pTestTask->fWrite)
364 {
365 /* @todo Do something sensible here. */
366 }
367 else
368 {
369 tstPDMACStressTestFileVerify(pTestFile, pTestTask); /* Will assert if it fails */
370 }
371
372 RTMemFree(pTestTask->DataSeg.pvSeg);
373 pTestTask->fActive = false;
374 AssertMsg(pTestFile->cTasksActiveCurr > 0, ("Trying to complete a non active task\n"));
375 ASMAtomicDecU32(&pTestFile->cTasksActiveCurr);
376}
377
378/**
379 * Sets up a test file creating the I/O thread.
380 *
381 * @returns VBox status code.
382 * @param pVM Pointer to the shared VM instance structure.
383 * @param pTestFile Pointer to the uninitialized test file structure.
384 * @param iTestId Unique test id.
385 */
386static int tstPDMACStressTestFileOpen(PVM pVM, PPDMACTESTFILE pTestFile, unsigned iTestId)
387{
388 int rc = VERR_NO_MEMORY;
389
390 /* Size is a multiple of 512 */
391 pTestFile->cbFileMax = RTRandU64Ex(FILE_SIZE_MIN, FILE_SIZE_MAX) & ~(511UL);
392 pTestFile->cbFileCurr = 0;
393 pTestFile->cbFileSegment = RTRandU32Ex(SEGMENT_SIZE_MIN, RT_MIN(pTestFile->cbFileMax, SEGMENT_SIZE_MAX)) & ~((size_t)511);
394
395 Assert(pTestFile->cbFileMax >= pTestFile->cbFileSegment);
396
397 /* Set up the segments array. */
398 pTestFile->cSegments = pTestFile->cbFileMax / pTestFile->cbFileSegment;
399 pTestFile->cSegments += ((pTestFile->cbFileMax % pTestFile->cbFileSegment) > 0) ? 1 : 0;
400
401 pTestFile->paSegs = (PPDMACTESTFILESEG)RTMemAllocZ(pTestFile->cSegments * sizeof(PDMACTESTFILESEG));
402 if (pTestFile->paSegs)
403 {
404 /* Init the segments */
405 for (unsigned i = 0; i < pTestFile->cSegments; i++)
406 {
407 PPDMACTESTFILESEG pSeg = &pTestFile->paSegs[i];
408
409 pSeg->off = (RTFOFF)i * pTestFile->cbFileSegment;
410 pSeg->cbSegment = pTestFile->cbFileSegment;
411
412 /* Let the buffer point to a random position in the test pattern. */
413 uint32_t offTestPattern = RTRandU64Ex(0, g_cbTestPattern - pSeg->cbSegment);
414
415 pSeg->pbData = g_pbTestPattern + offTestPattern;
416 }
417
418 /* Init task array. */
419 pTestFile->cTasksActiveMax = RTRandU32Ex(1, TASK_ACTIVE_MAX);
420 pTestFile->paTasks = (PPDMACTESTFILETASK)RTMemAllocZ(pTestFile->cTasksActiveMax * sizeof(PDMACTESTFILETASK));
421 if (pTestFile->paTasks)
422 {
423 /* Create the template */
424 char szDesc[256];
425
426 RTStrPrintf(szDesc, sizeof(szDesc), "Template-%d", iTestId);
427 rc = PDMR3AsyncCompletionTemplateCreateInternal(pVM, &pTestFile->pTemplate, tstPDMACStressTestFileTaskCompleted, pTestFile, szDesc);
428 if (RT_SUCCESS(rc))
429 {
430 /* Open the endpoint now. Because async completion endpoints cannot create files we have to do it before. */
431 char szFile[RTPATH_MAX];
432
433 RTStrPrintf(szFile, sizeof(szFile), "tstPDMAsyncCompletionStress-%d.tmp", iTestId);
434
435 RTFILE FileTmp;
436 rc = RTFileOpen(&FileTmp, szFile, RTFILE_O_CREATE | RTFILE_O_READWRITE);
437 if (RT_SUCCESS(rc))
438 {
439 RTFileClose(FileTmp);
440
441 rc = PDMR3AsyncCompletionEpCreateForFile(&pTestFile->hEndpoint, szFile, PDMACEP_FILE_FLAGS_CACHING, pTestFile->pTemplate);
442 if (RT_SUCCESS(rc))
443 {
444 char szThreadDesc[256];
445
446 pTestFile->fRunning = true;
447
448 /* Create the thread creating the I/O for the given file. */
449 RTStrPrintf(szThreadDesc, sizeof(szThreadDesc), "PDMACThread-%d", iTestId);
450 rc = PDMR3ThreadCreate(pVM, &pTestFile->hThread, pTestFile, tstPDMACTestFileThread,
451 NULL, 0, RTTHREADTYPE_IO, szThreadDesc);
452 if (RT_SUCCESS(rc))
453 {
454 RTPrintf(TESTCASE ": Created test file %s cbFileMax=%llu cbFileSegment=%u cSegments=%u cTasksActiveMax=%u\n",
455 szFile, pTestFile->cbFileMax, pTestFile->cbFileSegment, pTestFile->cSegments, pTestFile->cTasksActiveMax);
456 return VINF_SUCCESS;
457 }
458
459 PDMR3AsyncCompletionEpClose(pTestFile->hEndpoint);
460 }
461
462 RTFileDelete(szFile);
463 }
464
465 PDMR3AsyncCompletionTemplateDestroy(pTestFile->pTemplate);
466 }
467
468 RTMemFree(pTestFile->paTasks);
469 }
470 else
471 rc = VERR_NO_MEMORY;
472
473 RTMemFree(pTestFile->paSegs);
474 }
475 else
476 rc = VERR_NO_MEMORY;
477
478 RTPrintf(TESTCASE ": Opening test file with id %d failed rc=%Rrc\n", iTestId, rc);
479
480 return rc;
481}
482
483/**
484 * Closes a test file.
485 *
486 * @returns nothing.
487 * @param pTestFile Pointer to the test file.
488 */
489static void tstPDMACStressTestFileClose(PPDMACTESTFILE pTestFile)
490{
491 int rcThread;
492 int rc;
493
494 RTPrintf("Terminating I/O thread, please wait...\n");
495
496 /* Let the thread know that it should terminate. */
497 pTestFile->fRunning = false;
498
499 /* Wait for the thread to terminate. */
500 rc = PDMR3ThreadDestroy(pTestFile->hThread, &rcThread);
501
502 RTPrintf("Thread terminated with status code rc=%Rrc\n", rcThread);
503
504 /* Free resources */
505 RTMemFree(pTestFile->paTasks);
506 RTMemFree(pTestFile->paSegs);
507 PDMR3AsyncCompletionEpClose(pTestFile->hEndpoint);
508 PDMR3AsyncCompletionTemplateDestroy(pTestFile->pTemplate);
509}
510
511/**
512 * Inits the test pattern.
513 *
514 * @returns VBox status code.
515 */
516static int tstPDMACStressTestPatternInit(void)
517{
518 RTPrintf(TESTCASE ": Creating test pattern. Please wait...\n");
519 g_cbTestPattern = TEST_PATTERN_SIZE;
520 g_pbTestPattern = (uint8_t *)RTMemAlloc(g_cbTestPattern);
521 if (!g_pbTestPattern)
522 return VERR_NO_MEMORY;
523
524 RTRandBytes(g_pbTestPattern, g_cbTestPattern);
525 return VINF_SUCCESS;
526}
527
528static void tstPDMACStressTestPatternDestroy(void)
529{
530 RTPrintf(TESTCASE ": Destroying test pattern\n");
531 RTMemFree(g_pbTestPattern);
532}
533
534int main(int argc, char *argv[])
535{
536 int rcRet = 0; /* error count */
537 int rc = VINF_SUCCESS;
538
539 RTR3InitAndSUPLib();
540
541 PVM pVM;
542 rc = VMR3Create(1, NULL, NULL, NULL, NULL, &pVM);
543 if (RT_SUCCESS(rc))
544 {
545 /*
546 * Little hack to avoid the VM_ASSERT_EMT assertion.
547 */
548 RTTlsSet(pVM->pUVM->vm.s.idxTLS, &pVM->pUVM->aCpus[0]);
549 pVM->pUVM->aCpus[0].pUVM = pVM->pUVM;
550 pVM->pUVM->aCpus[0].vm.s.NativeThreadEMT = RTThreadNativeSelf();
551
552 rc = tstPDMACStressTestPatternInit();
553 if (RT_SUCCESS(rc))
554 {
555 unsigned cFilesOpened = 0;
556
557 /* Open the endpoints. */
558 for (cFilesOpened = 0; cFilesOpened < NR_OPEN_ENDPOINTS; cFilesOpened++)
559 {
560 rc = tstPDMACStressTestFileOpen(pVM, &g_aTestFiles[cFilesOpened], cFilesOpened);
561 if (RT_FAILURE(rc))
562 break;
563 }
564
565 if (RT_SUCCESS(rc))
566 {
567 /* Tests are running now. */
568 RTPrintf(TESTCASE ": Successfully opened all files. Running tests forever now or until an error is hit :)\n");
569 RTThreadSleep(RT_INDEFINITE_WAIT);
570 }
571
572 /* Close opened endpoints. */
573 for (unsigned i = 0; i < cFilesOpened; i++)
574 tstPDMACStressTestFileClose(&g_aTestFiles[i]);
575
576 tstPDMACStressTestPatternDestroy();
577 }
578 else
579 {
580 RTPrintf(TESTCASE ": failed to init test pattern!! rc=%Rrc\n", rc);
581 rcRet++;
582 }
583
584 rc = VMR3Destroy(pVM);
585 AssertMsg(rc == VINF_SUCCESS, ("%s: Destroying VM failed rc=%Rrc!!\n", __FUNCTION__, rc));
586 }
587 else
588 {
589 RTPrintf(TESTCASE ": failed to create VM!! rc=%Rrc\n", rc);
590 rcRet++;
591 }
592
593 return rcRet;
594}
595
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette