VirtualBox

source: vbox/trunk/src/VBox/Additions/linux/sharedfolders/regops.c@ 85689

Last change on this file since 85689 was 85689, checked in by vboxsync, 5 years ago

linux/vboxsf: Try workaround suse backporting get_user_pages_unlocked changes from 4.9 to their 4.4.x SEL12 releases.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 148.6 KB
Line 
1/* $Id: regops.c 85689 2020-08-11 12:48:15Z vboxsync $ */
2/** @file
3 * vboxsf - VBox Linux Shared Folders VFS, regular file inode and file operations.
4 */
5
6/*
7 * Copyright (C) 2006-2020 Oracle Corporation
8 *
9 * Permission is hereby granted, free of charge, to any person
10 * obtaining a copy of this software and associated documentation
11 * files (the "Software"), to deal in the Software without
12 * restriction, including without limitation the rights to use,
13 * copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following
16 * conditions:
17 *
18 * The above copyright notice and this permission notice shall be
19 * included in all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 * OTHER DEALINGS IN THE SOFTWARE.
29 */
30
31
32/*********************************************************************************************************************************
33* Header Files *
34*********************************************************************************************************************************/
35#include "vfsmod.h"
36#include <linux/uio.h>
37#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 32)
38# include <linux/aio.h> /* struct kiocb before 4.1 */
39#endif
40#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 12)
41# include <linux/buffer_head.h>
42#endif
43#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 12) \
44 && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
45# include <linux/writeback.h>
46#endif
47#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23) \
48 && LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
49# include <linux/splice.h>
50#endif
51#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 17) \
52 && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23)
53# include <linux/pipe_fs_i.h>
54#endif
55#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 10)
56# include <linux/swap.h> /* for mark_page_accessed */
57#endif
58#include <iprt/err.h>
59
60#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
61# define SEEK_END 2
62#endif
63
64#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
65# define iter_is_iovec(a_pIter) ( !((a_pIter)->type & ITER_KVEC) )
66#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0)
67# define iter_is_iovec(a_pIter) ( !((a_pIter)->type & (ITER_KVEC | ITER_BVEC)) )
68#endif
69
70#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)
71# define vm_fault_t int
72#endif
73
74#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 20)
75# define pgoff_t unsigned long
76#endif
77
78#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 12)
79# define PageUptodate(a_pPage) Page_Uptodate(a_pPage)
80#endif
81
82
83/*********************************************************************************************************************************
84* Structures and Typedefs *
85*********************************************************************************************************************************/
86#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
87struct vbsf_iov_iter {
88 unsigned int type;
89 unsigned int v_write : 1;
90 size_t iov_offset;
91 size_t nr_segs;
92 struct iovec const *iov;
93# ifdef VBOX_STRICT
94 struct iovec const *iov_org;
95 size_t nr_segs_org;
96# endif
97};
98# ifdef VBOX_STRICT
99# define VBSF_IOV_ITER_INITIALIZER(a_cSegs, a_pIov, a_fWrite) \
100 { vbsf_iov_iter_detect_type(a_pIov, a_cSegs), a_fWrite, 0, a_cSegs, a_pIov, a_pIov, a_cSegs }
101# else
102# define VBSF_IOV_ITER_INITIALIZER(a_cSegs, a_pIov, a_fWrite) \
103 { vbsf_iov_iter_detect_type(a_pIov, a_cSegs), a_fWrite, 0, a_cSegs, a_pIov }
104# endif
105# define ITER_KVEC 1
106# define iov_iter vbsf_iov_iter
107#endif
108
109#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
110/** Used by vbsf_iter_lock_pages() to keep the first page of the next segment. */
111struct vbsf_iter_stash {
112 struct page *pPage;
113 size_t off;
114 size_t cb;
115# if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
116 size_t offFromEnd;
117 struct iov_iter Copy;
118# endif
119};
120#endif /* >= 3.16.0 */
121/** Initializer for struct vbsf_iter_stash. */
122#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
123# define VBSF_ITER_STASH_INITIALIZER { NULL, 0 }
124#else
125# define VBSF_ITER_STASH_INITIALIZER { NULL, 0, ~(size_t)0 }
126#endif
127
128
129/*********************************************************************************************************************************
130* Internal Functions *
131*********************************************************************************************************************************/
132DECLINLINE(void) vbsf_put_page(struct page *pPage);
133static void vbsf_unlock_user_pages(struct page **papPages, size_t cPages, bool fSetDirty, bool fLockPgHack);
134static void vbsf_reg_write_sync_page_cache(struct address_space *mapping, loff_t offFile, uint32_t cbRange,
135 uint8_t const *pbSrcBuf, struct page **papSrcPages,
136 uint32_t offSrcPage, size_t cSrcPages);
137
138
139/*********************************************************************************************************************************
140* Provide more recent uio.h functionality to older kernels. *
141*********************************************************************************************************************************/
142#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
143
144/**
145 * Detects the vector type.
146 */
147static int vbsf_iov_iter_detect_type(struct iovec const *paIov, size_t cSegs)
148{
149 /* Check the first segment with a non-zero length. */
150 while (cSegs-- > 0) {
151 if (paIov->iov_len > 0) {
152 if (access_ok(VERIFY_READ, paIov->iov_base, paIov->iov_len))
153 return (uintptr_t)paIov->iov_base >= USER_DS.seg ? ITER_KVEC : 0;
154 AssertMsgFailed(("%p LB %#zx\n", paIov->iov_base, paIov->iov_len));
155 break;
156 }
157 paIov++;
158 }
159 return 0;
160}
161
162
163# undef iov_iter_count
164# define iov_iter_count(a_pIter) vbsf_iov_iter_count(a_pIter)
165static size_t vbsf_iov_iter_count(struct vbsf_iov_iter const *iter)
166{
167 size_t cbRet = 0;
168 size_t cLeft = iter->nr_segs;
169 struct iovec const *iov = iter->iov;
170 while (cLeft-- > 0) {
171 cbRet += iov->iov_len;
172 iov++;
173 }
174 return cbRet - iter->iov_offset;
175}
176
177
178# undef iov_iter_single_seg_count
179# define iov_iter_single_seg_count(a_pIter) vbsf_iov_iter_single_seg_count(a_pIter)
180static size_t vbsf_iov_iter_single_seg_count(struct vbsf_iov_iter const *iter)
181{
182 if (iter->nr_segs > 0)
183 return iter->iov->iov_len - iter->iov_offset;
184 return 0;
185}
186
187
188# undef iov_iter_advance
189# define iov_iter_advance(a_pIter, a_cbSkip) vbsf_iov_iter_advance(a_pIter, a_cbSkip)
190static void vbsf_iov_iter_advance(struct vbsf_iov_iter *iter, size_t cbSkip)
191{
192 SFLOG2(("vbsf_iov_iter_advance: cbSkip=%#zx\n", cbSkip));
193 if (iter->nr_segs > 0) {
194 size_t const cbLeftCur = iter->iov->iov_len - iter->iov_offset;
195 Assert(iter->iov_offset <= iter->iov->iov_len);
196 if (cbLeftCur > cbSkip) {
197 iter->iov_offset += cbSkip;
198 } else {
199 cbSkip -= cbLeftCur;
200 iter->iov_offset = 0;
201 iter->iov++;
202 iter->nr_segs--;
203 while (iter->nr_segs > 0) {
204 size_t const cbSeg = iter->iov->iov_len;
205 if (cbSeg > cbSkip) {
206 iter->iov_offset = cbSkip;
207 break;
208 }
209 cbSkip -= cbSeg;
210 iter->iov++;
211 iter->nr_segs--;
212 }
213 }
214 }
215}
216
217
218# undef iov_iter_get_pages
219# define iov_iter_get_pages(a_pIter, a_papPages, a_cbMax, a_cMaxPages, a_poffPg0) \
220 vbsf_iov_iter_get_pages(a_pIter, a_papPages, a_cbMax, a_cMaxPages, a_poffPg0)
221static ssize_t vbsf_iov_iter_get_pages(struct vbsf_iov_iter *iter, struct page **papPages,
222 size_t cbMax, unsigned cMaxPages, size_t *poffPg0)
223{
224 while (iter->nr_segs > 0) {
225 size_t const cbLeft = iter->iov->iov_len - iter->iov_offset;
226 Assert(iter->iov->iov_len >= iter->iov_offset);
227 if (cbLeft > 0) {
228 uintptr_t uPtrFrom = (uintptr_t)iter->iov->iov_base + iter->iov_offset;
229 size_t offPg0 = *poffPg0 = uPtrFrom & PAGE_OFFSET_MASK;
230 size_t cPagesLeft = RT_ALIGN_Z(offPg0 + cbLeft, PAGE_SIZE) >> PAGE_SHIFT;
231 size_t cPages = RT_MIN(cPagesLeft, cMaxPages);
232 struct task_struct *pTask = current;
233 size_t cPagesLocked;
234
235 down_read(&pTask->mm->mmap_sem);
236 cPagesLocked = get_user_pages(pTask, pTask->mm, uPtrFrom, cPages, iter->v_write, 1 /*force*/, papPages, NULL);
237 up_read(&pTask->mm->mmap_sem);
238 if (cPagesLocked == cPages) {
239 size_t cbRet = (cPages << PAGE_SHIFT) - offPg0;
240 if (cPages == cPagesLeft) {
241 size_t offLastPg = (uPtrFrom + cbLeft) & PAGE_OFFSET_MASK;
242 if (offLastPg)
243 cbRet -= PAGE_SIZE - offLastPg;
244 }
245 Assert(cbRet <= cbLeft);
246 return cbRet;
247 }
248 if (cPagesLocked > 0)
249 vbsf_unlock_user_pages(papPages, cPagesLocked, false /*fSetDirty*/, false /*fLockPgHack*/);
250 return -EFAULT;
251 }
252 iter->iov_offset = 0;
253 iter->iov++;
254 iter->nr_segs--;
255 }
256 AssertFailed();
257 return 0;
258}
259
260
261# undef iov_iter_truncate
262# define iov_iter_truncate(iter, cbNew) vbsf_iov_iter_truncate(iter, cbNew)
263static void vbsf_iov_iter_truncate(struct vbsf_iov_iter *iter, size_t cbNew)
264{
265 /* we have no counter or stuff, so it's a no-op. */
266 RT_NOREF(iter, cbNew);
267}
268
269
270# undef iov_iter_revert
271# define iov_iter_revert(a_pIter, a_cbRewind) vbsf_iov_iter_revert(a_pIter, a_cbRewind)
272void vbsf_iov_iter_revert(struct vbsf_iov_iter *iter, size_t cbRewind)
273{
274 SFLOG2(("vbsf_iov_iter_revert: cbRewind=%#zx\n", cbRewind));
275 if (iter->iov_offset > 0) {
276 if (cbRewind <= iter->iov_offset) {
277 iter->iov_offset -= cbRewind;
278 return;
279 }
280 cbRewind -= iter->iov_offset;
281 iter->iov_offset = 0;
282 }
283
284 while (cbRewind > 0) {
285 struct iovec const *pIov = --iter->iov;
286 size_t const cbSeg = pIov->iov_len;
287 iter->nr_segs++;
288
289 Assert((uintptr_t)pIov >= (uintptr_t)iter->iov_org);
290 Assert(iter->nr_segs <= iter->nr_segs_org);
291
292 if (cbRewind <= cbSeg) {
293 iter->iov_offset = cbSeg - cbRewind;
294 break;
295 }
296 cbRewind -= cbSeg;
297 }
298}
299
300#endif /* 2.6.19 <= linux < 3.16.0 */
301#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 35)
302
303/** This is for implementing cMaxPage on 3.16 which doesn't have it. */
304static ssize_t vbsf_iov_iter_get_pages_3_16(struct iov_iter *iter, struct page **papPages,
305 size_t cbMax, unsigned cMaxPages, size_t *poffPg0)
306{
307 if (!(iter->type & ITER_BVEC)) {
308 size_t const offPg0 = iter->iov_offset & PAGE_OFFSET_MASK;
309 size_t const cbMaxPages = ((size_t)cMaxPages << PAGE_SHIFT) - offPg0;
310 if (cbMax > cbMaxPages)
311 cbMax = cbMaxPages;
312 }
313 /* else: BVEC works a page at a time and shouldn't have much of a problem here. */
314 return iov_iter_get_pages(iter, papPages, cbMax, poffPg0);
315}
316# undef iov_iter_get_pages
317# define iov_iter_get_pages(a_pIter, a_papPages, a_cbMax, a_cMaxPages, a_poffPg0) \
318 vbsf_iov_iter_get_pages_3_16(a_pIter, a_papPages, a_cbMax, a_cMaxPages, a_poffPg0)
319
320#endif /* 3.16.0-3.16.34 */
321#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0)
322
323static size_t copy_from_iter(uint8_t *pbDst, size_t cbToCopy, struct iov_iter *pSrcIter)
324{
325 size_t const cbTotal = cbToCopy;
326 Assert(iov_iter_count(pSrcIter) >= cbToCopy);
327# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
328 if (pSrcIter->type & ITER_BVEC) {
329 while (cbToCopy > 0) {
330 size_t const offPage = (uintptr_t)pbDst & PAGE_OFFSET_MASK;
331 size_t const cbThisCopy = RT_MIN(PAGE_SIZE - offPage, cbToCopy);
332 struct page *pPage = rtR0MemObjLinuxVirtToPage(pbDst);
333 size_t cbCopied = copy_page_from_iter(pPage, offPage, cbThisCopy, pSrcIter);
334 AssertStmt(cbCopied <= cbThisCopy, cbCopied = cbThisCopy);
335 pbDst += cbCopied;
336 cbToCopy -= cbCopied;
337 if (cbCopied != cbToCopy)
338 break;
339 }
340 } else
341# endif
342 {
343 while (cbToCopy > 0) {
344 size_t cbThisCopy = iov_iter_single_seg_count(pSrcIter);
345 if (cbThisCopy > 0) {
346 if (cbThisCopy > cbToCopy)
347 cbThisCopy = cbToCopy;
348 if (pSrcIter->type & ITER_KVEC)
349 memcpy(pbDst, (void *)pSrcIter->iov->iov_base + pSrcIter->iov_offset, cbThisCopy);
350 else if (copy_from_user(pbDst, pSrcIter->iov->iov_base + pSrcIter->iov_offset, cbThisCopy) != 0)
351 break;
352 pbDst += cbThisCopy;
353 cbToCopy -= cbThisCopy;
354 }
355 iov_iter_advance(pSrcIter, cbThisCopy);
356 }
357 }
358 return cbTotal - cbToCopy;
359}
360
361
362static size_t copy_to_iter(uint8_t const *pbSrc, size_t cbToCopy, struct iov_iter *pDstIter)
363{
364 size_t const cbTotal = cbToCopy;
365 Assert(iov_iter_count(pDstIter) >= cbToCopy);
366# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
367 if (pDstIter->type & ITER_BVEC) {
368 while (cbToCopy > 0) {
369 size_t const offPage = (uintptr_t)pbSrc & PAGE_OFFSET_MASK;
370 size_t const cbThisCopy = RT_MIN(PAGE_SIZE - offPage, cbToCopy);
371 struct page *pPage = rtR0MemObjLinuxVirtToPage((void *)pbSrc);
372 size_t cbCopied = copy_page_to_iter(pPage, offPage, cbThisCopy, pDstIter);
373 AssertStmt(cbCopied <= cbThisCopy, cbCopied = cbThisCopy);
374 pbSrc += cbCopied;
375 cbToCopy -= cbCopied;
376 if (cbCopied != cbToCopy)
377 break;
378 }
379 } else
380# endif
381 {
382 while (cbToCopy > 0) {
383 size_t cbThisCopy = iov_iter_single_seg_count(pDstIter);
384 if (cbThisCopy > 0) {
385 if (cbThisCopy > cbToCopy)
386 cbThisCopy = cbToCopy;
387 if (pDstIter->type & ITER_KVEC)
388 memcpy((void *)pDstIter->iov->iov_base + pDstIter->iov_offset, pbSrc, cbThisCopy);
389 else if (copy_to_user(pDstIter->iov->iov_base + pDstIter->iov_offset, pbSrc, cbThisCopy) != 0) {
390 break;
391 }
392 pbSrc += cbThisCopy;
393 cbToCopy -= cbThisCopy;
394 }
395 iov_iter_advance(pDstIter, cbThisCopy);
396 }
397 }
398 return cbTotal - cbToCopy;
399}
400
401#endif /* 3.16.0 <= linux < 3.18.0 */
402
403
404
405/*********************************************************************************************************************************
406* Handle management *
407*********************************************************************************************************************************/
408
409/**
410 * Called when an inode is released to unlink all handles that might impossibly
411 * still be associated with it.
412 *
413 * @param pInodeInfo The inode which handles to drop.
414 */
415void vbsf_handle_drop_chain(struct vbsf_inode_info *pInodeInfo)
416{
417 struct vbsf_handle *pCur, *pNext;
418 unsigned long fSavedFlags;
419 SFLOGFLOW(("vbsf_handle_drop_chain: %p\n", pInodeInfo));
420 spin_lock_irqsave(&g_SfHandleLock, fSavedFlags);
421
422 RTListForEachSafe(&pInodeInfo->HandleList, pCur, pNext, struct vbsf_handle, Entry) {
423 AssertMsg( (pCur->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST))
424 == (VBSF_HANDLE_F_MAGIC | VBSF_HANDLE_F_ON_LIST), ("%p %#x\n", pCur, pCur->fFlags));
425 pCur->fFlags |= VBSF_HANDLE_F_ON_LIST;
426 RTListNodeRemove(&pCur->Entry);
427 }
428
429 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
430}
431
432
433/**
434 * Locates a handle that matches all the flags in @a fFlags.
435 *
436 * @returns Pointer to handle on success (retained), use vbsf_handle_release() to
437 * release it. NULL if no suitable handle was found.
438 * @param pInodeInfo The inode info to search.
439 * @param fFlagsSet The flags that must be set.
440 * @param fFlagsClear The flags that must be clear.
441 */
442struct vbsf_handle *vbsf_handle_find(struct vbsf_inode_info *pInodeInfo, uint32_t fFlagsSet, uint32_t fFlagsClear)
443{
444 struct vbsf_handle *pCur;
445 unsigned long fSavedFlags;
446 spin_lock_irqsave(&g_SfHandleLock, fSavedFlags);
447
448 RTListForEach(&pInodeInfo->HandleList, pCur, struct vbsf_handle, Entry) {
449 AssertMsg( (pCur->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST))
450 == (VBSF_HANDLE_F_MAGIC | VBSF_HANDLE_F_ON_LIST), ("%p %#x\n", pCur, pCur->fFlags));
451 if ((pCur->fFlags & (fFlagsSet | fFlagsClear)) == fFlagsSet) {
452 uint32_t cRefs = ASMAtomicIncU32(&pCur->cRefs);
453 if (cRefs > 1) {
454 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
455 SFLOGFLOW(("vbsf_handle_find: returns %p\n", pCur));
456 return pCur;
457 }
458 /* Oops, already being closed (safe as it's only ever increased here). */
459 ASMAtomicDecU32(&pCur->cRefs);
460 }
461 }
462
463 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
464 SFLOGFLOW(("vbsf_handle_find: returns NULL!\n"));
465 return NULL;
466}
467
468
469/**
470 * Slow worker for vbsf_handle_release() that does the freeing.
471 *
472 * @returns 0 (ref count).
473 * @param pHandle The handle to release.
474 * @param pSuperInfo The info structure for the shared folder associated with
475 * the handle.
476 * @param pszCaller The caller name (for logging failures).
477 */
478uint32_t vbsf_handle_release_slow(struct vbsf_handle *pHandle, struct vbsf_super_info *pSuperInfo, const char *pszCaller)
479{
480 int rc;
481 unsigned long fSavedFlags;
482
483 SFLOGFLOW(("vbsf_handle_release_slow: %p (%s)\n", pHandle, pszCaller));
484
485 /*
486 * Remove from the list.
487 */
488 spin_lock_irqsave(&g_SfHandleLock, fSavedFlags);
489
490 AssertMsg((pHandle->fFlags & VBSF_HANDLE_F_MAGIC_MASK) == VBSF_HANDLE_F_MAGIC, ("%p %#x\n", pHandle, pHandle->fFlags));
491 Assert(pHandle->pInodeInfo);
492 Assert(pHandle->pInodeInfo && pHandle->pInodeInfo->u32Magic == SF_INODE_INFO_MAGIC);
493
494 if (pHandle->fFlags & VBSF_HANDLE_F_ON_LIST) {
495 pHandle->fFlags &= ~VBSF_HANDLE_F_ON_LIST;
496 RTListNodeRemove(&pHandle->Entry);
497 }
498
499 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
500
501 /*
502 * Actually destroy it.
503 */
504 rc = VbglR0SfHostReqCloseSimple(pSuperInfo->map.root, pHandle->hHost);
505 if (RT_FAILURE(rc))
506 LogFunc(("Caller %s: VbglR0SfHostReqCloseSimple %#RX64 failed with rc=%Rrc\n", pszCaller, pHandle->hHost, rc));
507 pHandle->hHost = SHFL_HANDLE_NIL;
508 pHandle->fFlags = VBSF_HANDLE_F_MAGIC_DEAD;
509 kfree(pHandle);
510 return 0;
511}
512
513
514/**
515 * Appends a handle to a handle list.
516 *
517 * @param pInodeInfo The inode to add it to.
518 * @param pHandle The handle to add.
519 */
520void vbsf_handle_append(struct vbsf_inode_info *pInodeInfo, struct vbsf_handle *pHandle)
521{
522#ifdef VBOX_STRICT
523 struct vbsf_handle *pCur;
524#endif
525 unsigned long fSavedFlags;
526
527 SFLOGFLOW(("vbsf_handle_append: %p (to %p)\n", pHandle, pInodeInfo));
528 AssertMsg((pHandle->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST)) == VBSF_HANDLE_F_MAGIC,
529 ("%p %#x\n", pHandle, pHandle->fFlags));
530 Assert(pInodeInfo->u32Magic == SF_INODE_INFO_MAGIC);
531
532 spin_lock_irqsave(&g_SfHandleLock, fSavedFlags);
533
534 AssertMsg((pHandle->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST)) == VBSF_HANDLE_F_MAGIC,
535 ("%p %#x\n", pHandle, pHandle->fFlags));
536#ifdef VBOX_STRICT
537 RTListForEach(&pInodeInfo->HandleList, pCur, struct vbsf_handle, Entry) {
538 Assert(pCur != pHandle);
539 AssertMsg( (pCur->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST))
540 == (VBSF_HANDLE_F_MAGIC | VBSF_HANDLE_F_ON_LIST), ("%p %#x\n", pCur, pCur->fFlags));
541 }
542 pHandle->pInodeInfo = pInodeInfo;
543#endif
544
545 pHandle->fFlags |= VBSF_HANDLE_F_ON_LIST;
546 RTListAppend(&pInodeInfo->HandleList, &pHandle->Entry);
547
548 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
549}
550
551
552
553/*********************************************************************************************************************************
554* Misc *
555*********************************************************************************************************************************/
556
557#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 6)
558/** Any writable mappings? */
559DECLINLINE(bool) mapping_writably_mapped(struct address_space const *mapping)
560{
561# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 6)
562 return !list_empty(&mapping->i_mmap_shared);
563# else
564 return mapping->i_mmap_shared != NULL;
565# endif
566}
567#endif
568
569
570#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 12)
571/** Missing in 2.4.x, so just stub it for now. */
572DECLINLINE(bool) PageWriteback(struct page const *page)
573{
574 return false;
575}
576#endif
577
578
579/**
580 * Helper for deciding wheter we should do a read via the page cache or not.
581 *
582 * By default we will only use the page cache if there is a writable memory
583 * mapping of the file with a chance that it may have modified any of the pages
584 * already.
585 */
586DECLINLINE(bool) vbsf_should_use_cached_read(struct file *file, struct address_space *mapping, struct vbsf_super_info *pSuperInfo)
587{
588 if ( (file->f_flags & O_DIRECT)
589 || pSuperInfo->enmCacheMode == kVbsfCacheMode_None)
590 return false;
591 if ( pSuperInfo->enmCacheMode == kVbsfCacheMode_Read
592 || pSuperInfo->enmCacheMode == kVbsfCacheMode_ReadWrite)
593 return true;
594 Assert(pSuperInfo->enmCacheMode == kVbsfCacheMode_Strict);
595 return mapping
596 && mapping->nrpages > 0
597 && mapping_writably_mapped(mapping);
598}
599
600
601
602/*********************************************************************************************************************************
603* Pipe / splice stuff mainly for 2.6.17 >= linux < 2.6.31 (where no fallbacks were available) *
604*********************************************************************************************************************************/
605
606#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 17) \
607 && LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
608
609# if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
610# define LOCK_PIPE(a_pPipe) do { if ((a_pPipe)->inode) mutex_lock(&(a_pPipe)->inode->i_mutex); } while (0)
611# define UNLOCK_PIPE(a_pPipe) do { if ((a_pPipe)->inode) mutex_unlock(&(a_pPipe)->inode->i_mutex); } while (0)
612# else
613# define LOCK_PIPE(a_pPipe) pipe_lock(a_pPipe)
614# define UNLOCK_PIPE(a_pPipe) pipe_unlock(a_pPipe)
615# endif
616
617
618/** Waits for the pipe buffer status to change. */
619static void vbsf_wait_pipe(struct pipe_inode_info *pPipe)
620{
621 DEFINE_WAIT(WaitStuff);
622# ifdef TASK_NONINTERACTIVE
623 prepare_to_wait(&pPipe->wait, &WaitStuff, TASK_INTERRUPTIBLE | TASK_NONINTERACTIVE);
624# else
625 prepare_to_wait(&pPipe->wait, &WaitStuff, TASK_INTERRUPTIBLE);
626# endif
627 UNLOCK_PIPE(pPipe);
628
629 schedule();
630
631 finish_wait(&pPipe->wait, &WaitStuff);
632 LOCK_PIPE(pPipe);
633}
634
635
636/** Worker for vbsf_feed_pages_to_pipe that wakes up readers. */
637static void vbsf_wake_up_pipe(struct pipe_inode_info *pPipe, bool fReaders)
638{
639 smp_mb();
640 if (waitqueue_active(&pPipe->wait))
641 wake_up_interruptible_sync(&pPipe->wait);
642 if (fReaders)
643 kill_fasync(&pPipe->fasync_readers, SIGIO, POLL_IN);
644 else
645 kill_fasync(&pPipe->fasync_writers, SIGIO, POLL_OUT);
646}
647
648#endif
649#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 17) \
650 && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
651
652/** Verify pipe buffer content (needed for page-cache to ensure idle page). */
653static int vbsf_pipe_buf_confirm(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf)
654{
655 /*SFLOG3(("vbsf_pipe_buf_confirm: %p\n", pPipeBuf));*/
656 return 0;
657}
658
659
660/** Maps the buffer page. */
661static void *vbsf_pipe_buf_map(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf, int atomic)
662{
663 void *pvRet;
664 if (!atomic)
665 pvRet = kmap(pPipeBuf->page);
666 else {
667 pPipeBuf->flags |= PIPE_BUF_FLAG_ATOMIC;
668 pvRet = kmap_atomic(pPipeBuf->page, KM_USER0);
669 }
670 /*SFLOG3(("vbsf_pipe_buf_map: %p -> %p\n", pPipeBuf, pvRet));*/
671 return pvRet;
672}
673
674
675/** Unmaps the buffer page. */
676static void vbsf_pipe_buf_unmap(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf, void *pvMapping)
677{
678 /*SFLOG3(("vbsf_pipe_buf_unmap: %p/%p\n", pPipeBuf, pvMapping)); */
679 if (!(pPipeBuf->flags & PIPE_BUF_FLAG_ATOMIC))
680 kunmap(pPipeBuf->page);
681 else {
682 pPipeBuf->flags &= ~PIPE_BUF_FLAG_ATOMIC;
683 kunmap_atomic(pvMapping, KM_USER0);
684 }
685}
686
687
688/** Gets a reference to the page. */
689static void vbsf_pipe_buf_get(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf)
690{
691 page_cache_get(pPipeBuf->page);
692 /*SFLOG3(("vbsf_pipe_buf_get: %p (return count=%d)\n", pPipeBuf, page_count(pPipeBuf->page)));*/
693}
694
695
696/** Release the buffer page (counter to vbsf_pipe_buf_get). */
697static void vbsf_pipe_buf_release(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf)
698{
699 /*SFLOG3(("vbsf_pipe_buf_release: %p (incoming count=%d)\n", pPipeBuf, page_count(pPipeBuf->page)));*/
700 page_cache_release(pPipeBuf->page);
701}
702
703
704/** Attempt to steal the page.
705 * @returns 0 success, 1 on failure. */
706static int vbsf_pipe_buf_steal(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf)
707{
708 if (page_count(pPipeBuf->page) == 1) {
709 lock_page(pPipeBuf->page);
710 SFLOG3(("vbsf_pipe_buf_steal: %p -> 0\n", pPipeBuf));
711 return 0;
712 }
713 SFLOG3(("vbsf_pipe_buf_steal: %p -> 1\n", pPipeBuf));
714 return 1;
715}
716
717
718/**
719 * Pipe buffer operations for used by vbsf_feed_pages_to_pipe.
720 */
721static struct pipe_buf_operations vbsf_pipe_buf_ops = {
722 .can_merge = 0,
723# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
724 .confirm = vbsf_pipe_buf_confirm,
725# else
726 .pin = vbsf_pipe_buf_confirm,
727# endif
728 .map = vbsf_pipe_buf_map,
729 .unmap = vbsf_pipe_buf_unmap,
730 .get = vbsf_pipe_buf_get,
731 .release = vbsf_pipe_buf_release,
732 .steal = vbsf_pipe_buf_steal,
733};
734
735
736/**
737 * Feeds the pages to the pipe.
738 *
739 * Pages given to the pipe are set to NULL in papPages.
740 */
741static ssize_t vbsf_feed_pages_to_pipe(struct pipe_inode_info *pPipe, struct page **papPages, size_t cPages, uint32_t offPg0,
742 uint32_t cbActual, unsigned fFlags)
743{
744 ssize_t cbRet = 0;
745 size_t iPage = 0;
746 bool fNeedWakeUp = false;
747
748 LOCK_PIPE(pPipe);
749 for (;;) {
750 if ( pPipe->readers > 0
751 && pPipe->nrbufs < PIPE_BUFFERS) {
752 struct pipe_buffer *pPipeBuf = &pPipe->bufs[(pPipe->curbuf + pPipe->nrbufs) % PIPE_BUFFERS];
753 uint32_t const cbThisPage = RT_MIN(cbActual, PAGE_SIZE - offPg0);
754 pPipeBuf->len = cbThisPage;
755 pPipeBuf->offset = offPg0;
756# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
757 pPipeBuf->private = 0;
758# endif
759 pPipeBuf->ops = &vbsf_pipe_buf_ops;
760 pPipeBuf->flags = fFlags & SPLICE_F_GIFT ? PIPE_BUF_FLAG_GIFT : 0;
761 pPipeBuf->page = papPages[iPage];
762
763 papPages[iPage++] = NULL;
764 pPipe->nrbufs++;
765 fNeedWakeUp |= pPipe->inode != NULL;
766 offPg0 = 0;
767 cbRet += cbThisPage;
768
769 /* done? */
770 cbActual -= cbThisPage;
771 if (!cbActual)
772 break;
773 } else if (pPipe->readers == 0) {
774 SFLOGFLOW(("vbsf_feed_pages_to_pipe: no readers!\n"));
775 send_sig(SIGPIPE, current, 0);
776 if (cbRet == 0)
777 cbRet = -EPIPE;
778 break;
779 } else if (fFlags & SPLICE_F_NONBLOCK) {
780 if (cbRet == 0)
781 cbRet = -EAGAIN;
782 break;
783 } else if (signal_pending(current)) {
784 if (cbRet == 0)
785 cbRet = -ERESTARTSYS;
786 SFLOGFLOW(("vbsf_feed_pages_to_pipe: pending signal! (%zd)\n", cbRet));
787 break;
788 } else {
789 if (fNeedWakeUp) {
790 vbsf_wake_up_pipe(pPipe, true /*fReaders*/);
791 fNeedWakeUp = 0;
792 }
793 pPipe->waiting_writers++;
794 vbsf_wait_pipe(pPipe);
795 pPipe->waiting_writers--;
796 }
797 }
798 UNLOCK_PIPE(pPipe);
799
800 if (fNeedWakeUp)
801 vbsf_wake_up_pipe(pPipe, true /*fReaders*/);
802
803 return cbRet;
804}
805
806
807/**
808 * For splicing from a file to a pipe.
809 */
810static ssize_t vbsf_splice_read(struct file *file, loff_t *poffset, struct pipe_inode_info *pipe, size_t len, unsigned int flags)
811{
812 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
813 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
814 ssize_t cbRet;
815
816 SFLOGFLOW(("vbsf_splice_read: file=%p poffset=%p{%#RX64} pipe=%p len=%#zx flags=%#x\n", file, poffset, *poffset, pipe, len, flags));
817 if (vbsf_should_use_cached_read(file, inode->i_mapping, pSuperInfo)) {
818 cbRet = generic_file_splice_read(file, poffset, pipe, len, flags);
819 } else {
820 /*
821 * Create a read request.
822 */
823 loff_t offFile = *poffset;
824 size_t cPages = RT_MIN(RT_ALIGN_Z((offFile & ~PAGE_CACHE_MASK) + len, PAGE_CACHE_SIZE) >> PAGE_CACHE_SHIFT,
825 PIPE_BUFFERS);
826 VBOXSFREADPGLSTREQ *pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ,
827 PgLst.aPages[cPages]));
828 if (pReq) {
829 /*
830 * Allocate pages.
831 */
832 struct page *apPages[PIPE_BUFFERS];
833 size_t i;
834 pReq->PgLst.offFirstPage = (uint16_t)offFile & (uint16_t)PAGE_OFFSET_MASK;
835 cbRet = 0;
836 for (i = 0; i < cPages; i++) {
837 struct page *pPage;
838 apPages[i] = pPage = alloc_page(GFP_USER);
839 if (pPage) {
840 pReq->PgLst.aPages[i] = page_to_phys(pPage);
841# ifdef VBOX_STRICT
842 ASMMemFill32(kmap(pPage), PAGE_SIZE, UINT32_C(0xdeadbeef));
843 kunmap(pPage);
844# endif
845 } else {
846 cbRet = -ENOMEM;
847 break;
848 }
849 }
850 if (cbRet == 0) {
851 /*
852 * Do the reading.
853 */
854 uint32_t const cbToRead = RT_MIN((cPages << PAGE_SHIFT) - (offFile & PAGE_OFFSET_MASK), len);
855 struct vbsf_reg_info *sf_r = (struct vbsf_reg_info *)file->private_data;
856 int vrc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile, cbToRead, cPages);
857 if (RT_SUCCESS(vrc)) {
858 /*
859 * Get the number of bytes read, jettison the request
860 * and, in case of EOF, any unnecessary pages.
861 */
862 uint32_t cbActual = pReq->Parms.cb32Read.u.value32;
863 AssertStmt(cbActual <= cbToRead, cbActual = cbToRead);
864 SFLOG2(("vbsf_splice_read: read -> %#x bytes @ %#RX64\n", cbActual, offFile));
865
866 VbglR0PhysHeapFree(pReq);
867 pReq = NULL;
868
869 /*
870 * Now, feed it to the pipe thingy.
871 * This will take ownership of the all pages no matter what happens.
872 */
873 cbRet = vbsf_feed_pages_to_pipe(pipe, apPages, cPages, offFile & PAGE_OFFSET_MASK, cbActual, flags);
874 if (cbRet > 0)
875 *poffset = offFile + cbRet;
876 } else {
877 cbRet = -RTErrConvertToErrno(vrc);
878 SFLOGFLOW(("vbsf_splice_read: Read failed: %Rrc -> %zd\n", vrc, cbRet));
879 }
880 i = cPages;
881 }
882
883 while (i-- > 0)
884 if (apPages[i])
885 __free_pages(apPages[i], 0);
886 if (pReq)
887 VbglR0PhysHeapFree(pReq);
888 } else {
889 cbRet = -ENOMEM;
890 }
891 }
892 SFLOGFLOW(("vbsf_splice_read: returns %zd (%#zx), *poffset=%#RX64\n", cbRet, cbRet, *poffset));
893 return cbRet;
894}
895
896#endif /* 2.6.17 <= LINUX_VERSION_CODE < 2.6.31 */
897#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 17) \
898 && LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
899
900/**
901 * For splicing from a pipe to a file.
902 *
903 * Since we can combine buffers and request allocations, this should be faster
904 * than the default implementation.
905 */
906static ssize_t vbsf_splice_write(struct pipe_inode_info *pPipe, struct file *file, loff_t *poffset, size_t len, unsigned int flags)
907{
908 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
909 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
910 ssize_t cbRet;
911
912 SFLOGFLOW(("vbsf_splice_write: pPipe=%p file=%p poffset=%p{%#RX64} len=%#zx flags=%#x\n", pPipe, file, poffset, *poffset, len, flags));
913 /** @todo later if (false) {
914 cbRet = generic_file_splice_write(pPipe, file, poffset, len, flags);
915 } else */ {
916 /*
917 * Prepare a write request.
918 */
919# ifdef PIPE_BUFFERS
920 uint32_t const cMaxPages = RT_MIN(PIPE_BUFFERS, RT_ALIGN_Z(len, PAGE_SIZE) >> PAGE_SHIFT);
921# else
922 uint32_t const cMaxPages = RT_MIN(RT_MAX(RT_MIN(pPipe->buffers, 256), PIPE_DEF_BUFFERS),
923 RT_ALIGN_Z(len, PAGE_SIZE) >> PAGE_SHIFT);
924# endif
925 VBOXSFWRITEPGLSTREQ *pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ,
926 PgLst.aPages[cMaxPages]));
927 if (pReq) {
928 /*
929 * Feed from the pipe.
930 */
931 struct vbsf_reg_info *sf_r = (struct vbsf_reg_info *)file->private_data;
932 struct address_space *mapping = inode->i_mapping;
933 loff_t offFile = *poffset;
934 bool fNeedWakeUp = false;
935 cbRet = 0;
936
937 LOCK_PIPE(pPipe);
938
939 for (;;) {
940 unsigned cBufs = pPipe->nrbufs;
941 /*SFLOG2(("vbsf_splice_write: nrbufs=%#x curbuf=%#x\n", cBufs, pPipe->curbuf));*/
942 if (cBufs) {
943 /*
944 * There is data available. Write it to the file.
945 */
946 int vrc;
947 struct pipe_buffer *pPipeBuf = &pPipe->bufs[pPipe->curbuf];
948 uint32_t cPagesToWrite = 1;
949 uint32_t cbToWrite = pPipeBuf->len;
950
951 Assert(pPipeBuf->offset < PAGE_SIZE);
952 Assert(pPipeBuf->offset + pPipeBuf->len <= PAGE_SIZE);
953
954 pReq->PgLst.offFirstPage = pPipeBuf->offset & PAGE_OFFSET;
955 pReq->PgLst.aPages[0] = page_to_phys(pPipeBuf->page);
956
957 /* Add any adjacent page buffers: */
958 while ( cPagesToWrite < cBufs
959 && cPagesToWrite < cMaxPages
960 && ((pReq->PgLst.offFirstPage + cbToWrite) & PAGE_OFFSET_MASK) == 0) {
961# ifdef PIPE_BUFFERS
962 struct pipe_buffer *pPipeBuf2 = &pPipe->bufs[(pPipe->curbuf + cPagesToWrite) % PIPE_BUFFERS];
963# else
964 struct pipe_buffer *pPipeBuf2 = &pPipe->bufs[(pPipe->curbuf + cPagesToWrite) % pPipe->buffers];
965# endif
966 Assert(pPipeBuf2->len <= PAGE_SIZE);
967 Assert(pPipeBuf2->offset < PAGE_SIZE);
968 if (pPipeBuf2->offset != 0)
969 break;
970 pReq->PgLst.aPages[cPagesToWrite] = page_to_phys(pPipeBuf2->page);
971 cbToWrite += pPipeBuf2->len;
972 cPagesToWrite += 1;
973 }
974
975 /* Check that we don't have signals pending before we issue the write, as
976 we'll only end up having to cancel the HGCM request 99% of the time: */
977 if (!signal_pending(current)) {
978 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
979 vrc = VbglR0SfHostReqWritePgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile,
980 cbToWrite, cPagesToWrite);
981 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
982 } else
983 vrc = VERR_INTERRUPTED;
984 if (RT_SUCCESS(vrc)) {
985 /*
986 * Get the number of bytes actually written, update file position
987 * and return value, and advance the pipe buffer.
988 */
989 uint32_t cbActual = pReq->Parms.cb32Write.u.value32;
990 AssertStmt(cbActual <= cbToWrite, cbActual = cbToWrite);
991 SFLOG2(("vbsf_splice_write: write -> %#x bytes @ %#RX64\n", cbActual, offFile));
992
993 cbRet += cbActual;
994
995 while (cbActual > 0) {
996 uint32_t cbAdvance = RT_MIN(pPipeBuf->len, cbActual);
997
998 vbsf_reg_write_sync_page_cache(mapping, offFile, cbAdvance, NULL,
999 &pPipeBuf->page, pPipeBuf->offset, 1);
1000
1001 offFile += cbAdvance;
1002 cbActual -= cbAdvance;
1003 pPipeBuf->offset += cbAdvance;
1004 pPipeBuf->len -= cbAdvance;
1005
1006 if (!pPipeBuf->len) {
1007 struct pipe_buf_operations const *pOps = pPipeBuf->ops;
1008 pPipeBuf->ops = NULL;
1009 pOps->release(pPipe, pPipeBuf);
1010
1011# ifdef PIPE_BUFFERS
1012 pPipe->curbuf = (pPipe->curbuf + 1) % PIPE_BUFFERS;
1013# else
1014 pPipe->curbuf = (pPipe->curbuf + 1) % pPipe->buffers;
1015# endif
1016 pPipe->nrbufs -= 1;
1017 pPipeBuf = &pPipe->bufs[pPipe->curbuf];
1018
1019# if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
1020 fNeedWakeUp |= pPipe->inode != NULL;
1021# else
1022 fNeedWakeUp = true;
1023# endif
1024 } else {
1025 Assert(cbActual == 0);
1026 break;
1027 }
1028 }
1029
1030 *poffset = offFile;
1031 } else {
1032 if (cbRet == 0)
1033 cbRet = vrc == VERR_INTERRUPTED ? -ERESTARTSYS : -RTErrConvertToErrno(vrc);
1034 SFLOGFLOW(("vbsf_splice_write: Write failed: %Rrc -> %zd (cbRet=%#zx)\n",
1035 vrc, -RTErrConvertToErrno(vrc), cbRet));
1036 break;
1037 }
1038 } else {
1039 /*
1040 * Wait for data to become available, if there is chance that'll happen.
1041 */
1042 /* Quit if there are no writers (think EOF): */
1043 if (pPipe->writers == 0) {
1044 SFLOGFLOW(("vbsf_splice_write: No buffers. No writers. The show is done!\n"));
1045 break;
1046 }
1047
1048 /* Quit if if we've written some and no writers waiting on the lock: */
1049 if (cbRet > 0 && pPipe->waiting_writers == 0) {
1050 SFLOGFLOW(("vbsf_splice_write: No waiting writers, returning what we've got.\n"));
1051 break;
1052 }
1053
1054 /* Quit with EAGAIN if non-blocking: */
1055 if (flags & SPLICE_F_NONBLOCK) {
1056 if (cbRet == 0)
1057 cbRet = -EAGAIN;
1058 break;
1059 }
1060
1061 /* Quit if we've got pending signals: */
1062 if (signal_pending(current)) {
1063 if (cbRet == 0)
1064 cbRet = -ERESTARTSYS;
1065 SFLOGFLOW(("vbsf_splice_write: pending signal! (%zd)\n", cbRet));
1066 break;
1067 }
1068
1069 /* Wake up writers before we start waiting: */
1070 if (fNeedWakeUp) {
1071 vbsf_wake_up_pipe(pPipe, false /*fReaders*/);
1072 fNeedWakeUp = false;
1073 }
1074 vbsf_wait_pipe(pPipe);
1075 }
1076 } /* feed loop */
1077
1078 if (fNeedWakeUp)
1079 vbsf_wake_up_pipe(pPipe, false /*fReaders*/);
1080
1081 UNLOCK_PIPE(pPipe);
1082
1083 VbglR0PhysHeapFree(pReq);
1084 } else {
1085 cbRet = -ENOMEM;
1086 }
1087 }
1088 SFLOGFLOW(("vbsf_splice_write: returns %zd (%#zx), *poffset=%#RX64\n", cbRet, cbRet, *poffset));
1089 return cbRet;
1090}
1091
1092#endif /* 2.6.17 <= LINUX_VERSION_CODE < 3.16.0 */
1093
1094#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 30) \
1095 && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23)
1096/**
1097 * Our own senfile implementation that does not go via the page cache like
1098 * generic_file_sendfile() does.
1099 */
1100static ssize_t vbsf_reg_sendfile(struct file *pFile, loff_t *poffFile, size_t cbToSend, read_actor_t pfnActor,
1101# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 8)
1102 void *pvUser
1103# else
1104 void __user *pvUser
1105# endif
1106 )
1107{
1108 struct inode *inode = VBSF_GET_F_DENTRY(pFile)->d_inode;
1109 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
1110 ssize_t cbRet;
1111 SFLOGFLOW(("vbsf_reg_sendfile: pFile=%p poffFile=%p{%#RX64} cbToSend=%#zx pfnActor=%p pvUser=%p\n",
1112 pFile, poffFile, poffFile ? *poffFile : 0, cbToSend, pfnActor, pvUser));
1113 Assert(pSuperInfo);
1114
1115 /*
1116 * Return immediately if asked to send nothing.
1117 */
1118 if (cbToSend == 0)
1119 return 0;
1120
1121 /*
1122 * Like for vbsf_reg_read() and vbsf_reg_read_iter(), we allow going via
1123 * the page cache in some cases or configs.
1124 */
1125 if (vbsf_should_use_cached_read(pFile, inode->i_mapping, pSuperInfo)) {
1126 cbRet = generic_file_sendfile(pFile, poffFile, cbToSend, pfnActor, pvUser);
1127 SFLOGFLOW(("vbsf_reg_sendfile: returns %#zx *poffFile=%#RX64 [generic_file_sendfile]\n", cbRet, poffFile ? *poffFile : UINT64_MAX));
1128 } else {
1129 /*
1130 * Allocate a request and a bunch of pages for reading from the file.
1131 */
1132 struct page *apPages[16];
1133 loff_t offFile = poffFile ? *poffFile : 0;
1134 size_t const cPages = cbToSend + ((size_t)offFile & PAGE_OFFSET_MASK) >= RT_ELEMENTS(apPages) * PAGE_SIZE
1135 ? RT_ELEMENTS(apPages)
1136 : RT_ALIGN_Z(cbToSend + ((size_t)offFile & PAGE_OFFSET_MASK), PAGE_SIZE) >> PAGE_SHIFT;
1137 size_t iPage;
1138 VBOXSFREADPGLSTREQ *pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ,
1139 PgLst.aPages[cPages]));
1140 if (pReq) {
1141 Assert(cPages > 0);
1142 cbRet = 0;
1143 for (iPage = 0; iPage < cPages; iPage++) {
1144 struct page *pPage;
1145 apPages[iPage] = pPage = alloc_page(GFP_USER);
1146 if (pPage) {
1147 Assert(page_count(pPage) == 1);
1148 pReq->PgLst.aPages[iPage] = page_to_phys(pPage);
1149 } else {
1150 while (iPage-- > 0)
1151 vbsf_put_page(apPages[iPage]);
1152 cbRet = -ENOMEM;
1153 break;
1154 }
1155 }
1156 if (cbRet == 0) {
1157 /*
1158 * Do the job.
1159 */
1160 struct vbsf_reg_info *sf_r = (struct vbsf_reg_info *)pFile->private_data;
1161 read_descriptor_t RdDesc;
1162 RdDesc.count = cbToSend;
1163# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 8)
1164 RdDesc.arg.data = pvUser;
1165# else
1166 RdDesc.buf = pvUser;
1167# endif
1168 RdDesc.written = 0;
1169 RdDesc.error = 0;
1170
1171 Assert(sf_r);
1172 Assert((sf_r->Handle.fFlags & VBSF_HANDLE_F_MAGIC_MASK) == VBSF_HANDLE_F_MAGIC);
1173
1174 while (cbToSend > 0) {
1175 /*
1176 * Read another chunk. For paranoid reasons, we keep data where the page cache
1177 * would keep it, i.e. page offset bits corresponds to the file offset bits.
1178 */
1179 uint32_t const offPg0 = (uint32_t)offFile & (uint32_t)PAGE_OFFSET_MASK;
1180 uint32_t const cbToRead = RT_MIN((cPages << PAGE_SHIFT) - offPg0, cbToSend);
1181 uint32_t const cPagesToRead = RT_ALIGN_Z(cbToRead + offPg0, PAGE_SIZE) >> PAGE_SHIFT;
1182 int vrc;
1183 pReq->PgLst.offFirstPage = (uint16_t)offPg0;
1184 if (!signal_pending(current))
1185 vrc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile,
1186 cbToRead, cPagesToRead);
1187 else
1188 vrc = VERR_INTERRUPTED;
1189 if (RT_SUCCESS(vrc)) {
1190 /*
1191 * Pass what we read to the actor.
1192 */
1193 uint32_t off = offPg0;
1194 uint32_t cbActual = pReq->Parms.cb32Read.u.value32;
1195 bool const fIsEof = cbActual < cbToRead;
1196 AssertStmt(cbActual <= cbToRead, cbActual = cbToRead);
1197 SFLOG3(("vbsf_reg_sendfile: Read %#x bytes (offPg0=%#x), wanted %#x ...\n", cbActual, offPg0, cbToRead));
1198
1199 iPage = 0;
1200 while (cbActual > 0) {
1201 uint32_t const cbPage = RT_MIN(cbActual, PAGE_SIZE - off);
1202 int const cbRetActor = pfnActor(&RdDesc, apPages[iPage], off, cbPage);
1203 Assert(cbRetActor >= 0); /* Returns zero on failure, with RdDesc.error holding the status code. */
1204
1205 AssertMsg(iPage < cPages && iPage < cPagesToRead, ("iPage=%#x cPages=%#x cPagesToRead=%#x\n", iPage, cPages, cPagesToRead));
1206
1207 offFile += cbRetActor;
1208 if ((uint32_t)cbRetActor == cbPage && RdDesc.count > 0) {
1209 cbActual -= cbPage;
1210 cbToSend -= cbPage;
1211 iPage++;
1212 } else {
1213 SFLOG3(("vbsf_reg_sendfile: cbRetActor=%#x (%d) cbPage=%#x RdDesc{count=%#lx error=%d} iPage=%#x/%#x/%#x cbToSend=%#zx\n",
1214 cbRetActor, cbRetActor, cbPage, RdDesc.count, RdDesc.error, iPage, cPagesToRead, cPages, cbToSend));
1215 vrc = VERR_CALLBACK_RETURN;
1216 break;
1217 }
1218 off = 0;
1219 }
1220
1221 /*
1222 * Are we done yet?
1223 */
1224 if (RT_FAILURE_NP(vrc) || cbToSend == 0 || RdDesc.error != 0 || fIsEof) {
1225 break;
1226 }
1227
1228 /*
1229 * Replace pages held by the actor.
1230 */
1231 vrc = VINF_SUCCESS;
1232 for (iPage = 0; iPage < cPages; iPage++) {
1233 struct page *pPage = apPages[iPage];
1234 if (page_count(pPage) != 1) {
1235 struct page *pNewPage = alloc_page(GFP_USER);
1236 if (pNewPage) {
1237 SFLOGFLOW(("vbsf_reg_sendfile: Replacing page #%x: %p -> %p\n", iPage, pPage, pNewPage));
1238 vbsf_put_page(pPage);
1239 apPages[iPage] = pNewPage;
1240 } else {
1241 SFLOGFLOW(("vbsf_reg_sendfile: Failed to allocate a replacement page.\n"));
1242 vrc = VERR_NO_MEMORY;
1243 break;
1244 }
1245 }
1246 }
1247 if (RT_FAILURE(vrc))
1248 break; /* RdDesc.written should be non-zero, so don't bother with setting error. */
1249 } else {
1250 RdDesc.error = vrc == VERR_INTERRUPTED ? -ERESTARTSYS : -RTErrConvertToErrno(vrc);
1251 SFLOGFLOW(("vbsf_reg_sendfile: Read failed: %Rrc -> %zd (RdDesc.error=%#d)\n",
1252 vrc, -RTErrConvertToErrno(vrc), RdDesc.error));
1253 break;
1254 }
1255 }
1256
1257 /*
1258 * Free memory.
1259 */
1260 for (iPage = 0; iPage < cPages; iPage++)
1261 vbsf_put_page(apPages[iPage]);
1262
1263 /*
1264 * Set the return values.
1265 */
1266 if (RdDesc.written) {
1267 cbRet = RdDesc.written;
1268 if (poffFile)
1269 *poffFile = offFile;
1270 } else {
1271 cbRet = RdDesc.error;
1272 }
1273 }
1274 VbglR0PhysHeapFree(pReq);
1275 } else {
1276 cbRet = -ENOMEM;
1277 }
1278 SFLOGFLOW(("vbsf_reg_sendfile: returns %#zx offFile=%#RX64\n", cbRet, offFile));
1279 }
1280 return cbRet;
1281}
1282#endif /* 2.5.30 <= LINUX_VERSION_CODE < 2.6.23 */
1283
1284
1285/*********************************************************************************************************************************
1286* File operations on regular files *
1287*********************************************************************************************************************************/
1288
1289/** Wrapper around put_page / page_cache_release. */
1290DECLINLINE(void) vbsf_put_page(struct page *pPage)
1291{
1292#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
1293 put_page(pPage);
1294#else
1295 page_cache_release(pPage);
1296#endif
1297}
1298
1299
1300/** Wrapper around get_page / page_cache_get. */
1301DECLINLINE(void) vbsf_get_page(struct page *pPage)
1302{
1303#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
1304 get_page(pPage);
1305#else
1306 page_cache_get(pPage);
1307#endif
1308}
1309
1310
1311/** Companion to vbsf_lock_user_pages(). */
1312static void vbsf_unlock_user_pages(struct page **papPages, size_t cPages, bool fSetDirty, bool fLockPgHack)
1313{
1314 /* We don't mark kernel pages dirty: */
1315 if (fLockPgHack)
1316 fSetDirty = false;
1317
1318 while (cPages-- > 0)
1319 {
1320 struct page *pPage = papPages[cPages];
1321 Assert((ssize_t)cPages >= 0);
1322 if (fSetDirty && !PageReserved(pPage))
1323 set_page_dirty(pPage);
1324 vbsf_put_page(pPage);
1325 }
1326}
1327
1328
1329/**
1330 * Worker for vbsf_lock_user_pages_failed_check_kernel() and
1331 * vbsf_iter_lock_pages().
1332 */
1333static int vbsf_lock_kernel_pages(uint8_t *pbStart, bool fWrite, size_t cPages, struct page **papPages)
1334{
1335 uintptr_t const uPtrFrom = (uintptr_t)pbStart;
1336 uintptr_t const uPtrLast = (uPtrFrom & ~(uintptr_t)PAGE_OFFSET_MASK) + (cPages << PAGE_SHIFT) - 1;
1337 uint8_t *pbPage = (uint8_t *)uPtrLast;
1338 size_t iPage = cPages;
1339
1340 /*
1341 * Touch the pages first (paranoia^2).
1342 */
1343 if (fWrite) {
1344 uint8_t volatile *pbProbe = (uint8_t volatile *)uPtrFrom;
1345 while (iPage-- > 0) {
1346 *pbProbe = *pbProbe;
1347 pbProbe += PAGE_SIZE;
1348 }
1349 } else {
1350 uint8_t const *pbProbe = (uint8_t const *)uPtrFrom;
1351 while (iPage-- > 0) {
1352 ASMProbeReadByte(pbProbe);
1353 pbProbe += PAGE_SIZE;
1354 }
1355 }
1356
1357 /*
1358 * Get the pages.
1359 * Note! Fixes here probably applies to rtR0MemObjNativeLockKernel as well.
1360 */
1361 iPage = cPages;
1362 if ( uPtrFrom >= (unsigned long)__va(0)
1363 && uPtrLast < (unsigned long)high_memory) {
1364 /* The physical page mapping area: */
1365 while (iPage-- > 0) {
1366 struct page *pPage = papPages[iPage] = virt_to_page(pbPage);
1367 vbsf_get_page(pPage);
1368 pbPage -= PAGE_SIZE;
1369 }
1370 } else {
1371 /* This is vmalloc or some such thing, so go thru page tables: */
1372 while (iPage-- > 0) {
1373 struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1374 if (pPage) {
1375 papPages[iPage] = pPage;
1376 vbsf_get_page(pPage);
1377 pbPage -= PAGE_SIZE;
1378 } else {
1379 while (++iPage < cPages) {
1380 pPage = papPages[iPage];
1381 vbsf_put_page(pPage);
1382 }
1383 return -EFAULT;
1384 }
1385 }
1386 }
1387 return 0;
1388}
1389
1390
1391/**
1392 * Catches kernel_read() and kernel_write() calls and works around them.
1393 *
1394 * The file_operations::read and file_operations::write callbacks supposedly
1395 * hands us the user buffers to read into and write out of. To allow the kernel
1396 * to read and write without allocating buffers in userland, they kernel_read()
1397 * and kernel_write() increases the user space address limit before calling us
1398 * so that copyin/copyout won't reject it. Our problem is that get_user_pages()
1399 * works on the userspace address space structures and will not be fooled by an
1400 * increased addr_limit.
1401 *
1402 * This code tries to detect this situation and fake get_user_lock() for the
1403 * kernel buffer.
1404 */
1405static int vbsf_lock_user_pages_failed_check_kernel(uintptr_t uPtrFrom, size_t cPages, bool fWrite, int rcFailed,
1406 struct page **papPages, bool *pfLockPgHack)
1407{
1408 /*
1409 * Check that this is valid user memory that is actually in the kernel range.
1410 */
1411#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) || defined(RHEL_81)
1412 if ( access_ok((void *)uPtrFrom, cPages << PAGE_SHIFT)
1413 && uPtrFrom >= USER_DS.seg)
1414#else
1415 if ( access_ok(fWrite ? VERIFY_WRITE : VERIFY_READ, (void *)uPtrFrom, cPages << PAGE_SHIFT)
1416 && uPtrFrom >= USER_DS.seg)
1417#endif
1418 {
1419 int rc = vbsf_lock_kernel_pages((uint8_t *)uPtrFrom, fWrite, cPages, papPages);
1420 if (rc == 0) {
1421 *pfLockPgHack = true;
1422 return 0;
1423 }
1424 }
1425
1426 return rcFailed;
1427}
1428
1429
1430/** Wrapper around get_user_pages. */
1431DECLINLINE(int) vbsf_lock_user_pages(uintptr_t uPtrFrom, size_t cPages, bool fWrite, struct page **papPages, bool *pfLockPgHack)
1432{
1433# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) \
1434 || ( defined(CONFIG_SUSE_KERNEL) /** @todo Figure out when exactly. Also, guessing a bit here what got backported. */ \
1435 && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 73) \
1436 && LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0))
1437 ssize_t cPagesLocked = get_user_pages_unlocked(uPtrFrom, cPages, papPages,
1438 fWrite ? FOLL_WRITE | FOLL_FORCE : FOLL_FORCE);
1439# elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
1440 ssize_t cPagesLocked = get_user_pages_unlocked(uPtrFrom, cPages, fWrite, 1 /*force*/, papPages);
1441# elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 168) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
1442 ssize_t cPagesLocked = get_user_pages_unlocked(current, current->mm, uPtrFrom, cPages, papPages,
1443 fWrite ? FOLL_WRITE | FOLL_FORCE : FOLL_FORCE);
1444# elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0)
1445 ssize_t cPagesLocked = get_user_pages_unlocked(current, current->mm, uPtrFrom, cPages, fWrite, 1 /*force*/, papPages);
1446# else
1447 struct task_struct *pTask = current;
1448 ssize_t cPagesLocked;
1449 down_read(&pTask->mm->mmap_sem);
1450 cPagesLocked = get_user_pages(pTask, pTask->mm, uPtrFrom, cPages, fWrite, 1 /*force*/, papPages, NULL);
1451 up_read(&pTask->mm->mmap_sem);
1452# endif
1453 *pfLockPgHack = false;
1454 if (cPagesLocked == cPages)
1455 return 0;
1456
1457 /*
1458 * It failed.
1459 */
1460 if (cPagesLocked < 0)
1461 return vbsf_lock_user_pages_failed_check_kernel(uPtrFrom, cPages, fWrite, (int)cPagesLocked, papPages, pfLockPgHack);
1462
1463 vbsf_unlock_user_pages(papPages, cPagesLocked, false /*fSetDirty*/, false /*fLockPgHack*/);
1464
1465 /* We could use uPtrFrom + cPagesLocked to get the correct status here... */
1466 return -EFAULT;
1467}
1468
1469
1470/**
1471 * Read function used when accessing files that are memory mapped.
1472 *
1473 * We read from the page cache here to present the a cohertent picture of the
1474 * the file content.
1475 */
1476static ssize_t vbsf_reg_read_mapped(struct file *file, char /*__user*/ *buf, size_t size, loff_t *off)
1477{
1478#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
1479 struct iovec iov = { .iov_base = buf, .iov_len = size };
1480 struct iov_iter iter;
1481 struct kiocb kiocb;
1482 ssize_t cbRet;
1483
1484 init_sync_kiocb(&kiocb, file);
1485 kiocb.ki_pos = *off;
1486 iov_iter_init(&iter, READ, &iov, 1, size);
1487
1488 cbRet = generic_file_read_iter(&kiocb, &iter);
1489
1490 *off = kiocb.ki_pos;
1491 return cbRet;
1492
1493#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
1494 struct iovec iov = { .iov_base = buf, .iov_len = size };
1495 struct kiocb kiocb;
1496 ssize_t cbRet;
1497
1498 init_sync_kiocb(&kiocb, file);
1499 kiocb.ki_pos = *off;
1500
1501 cbRet = generic_file_aio_read(&kiocb, &iov, 1, *off);
1502 if (cbRet == -EIOCBQUEUED)
1503 cbRet = wait_on_sync_kiocb(&kiocb);
1504
1505 *off = kiocb.ki_pos;
1506 return cbRet;
1507
1508#else /* 2.6.18 or earlier: */
1509 return generic_file_read(file, buf, size, off);
1510#endif
1511}
1512
1513
1514/**
1515 * Fallback case of vbsf_reg_read() that locks the user buffers and let the host
1516 * write directly to them.
1517 */
1518static ssize_t vbsf_reg_read_locking(struct file *file, char /*__user*/ *buf, size_t size, loff_t *off,
1519 struct vbsf_super_info *pSuperInfo, struct vbsf_reg_info *sf_r)
1520{
1521 /*
1522 * Lock pages and execute the read, taking care not to pass the host
1523 * more than it can handle in one go or more than we care to allocate
1524 * page arrays for. The latter limit is set at just short of 32KB due
1525 * to how the physical heap works.
1526 */
1527 struct page *apPagesStack[16];
1528 struct page **papPages = &apPagesStack[0];
1529 struct page **papPagesFree = NULL;
1530 VBOXSFREADPGLSTREQ *pReq;
1531 loff_t offFile = *off;
1532 ssize_t cbRet = -ENOMEM;
1533 size_t cPages = (((uintptr_t)buf & PAGE_OFFSET_MASK) + size + PAGE_OFFSET_MASK) >> PAGE_SHIFT;
1534 size_t cMaxPages = RT_MIN(RT_MAX(pSuperInfo->cMaxIoPages, 1), cPages);
1535 bool fLockPgHack;
1536
1537 pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ, PgLst.aPages[cMaxPages]));
1538 while (!pReq && cMaxPages > 4) {
1539 cMaxPages /= 2;
1540 pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ, PgLst.aPages[cMaxPages]));
1541 }
1542 if (pReq && cMaxPages > RT_ELEMENTS(apPagesStack))
1543 papPagesFree = papPages = kmalloc(cMaxPages * sizeof(sizeof(papPages[0])), GFP_KERNEL);
1544 if (pReq && papPages) {
1545 cbRet = 0;
1546 for (;;) {
1547 /*
1548 * Figure out how much to process now and lock the user pages.
1549 */
1550 int rc;
1551 size_t cbChunk = (uintptr_t)buf & PAGE_OFFSET_MASK;
1552 pReq->PgLst.offFirstPage = (uint16_t)cbChunk;
1553 cPages = RT_ALIGN_Z(cbChunk + size, PAGE_SIZE) >> PAGE_SHIFT;
1554 if (cPages <= cMaxPages)
1555 cbChunk = size;
1556 else {
1557 cPages = cMaxPages;
1558 cbChunk = (cMaxPages << PAGE_SHIFT) - cbChunk;
1559 }
1560
1561 rc = vbsf_lock_user_pages((uintptr_t)buf, cPages, true /*fWrite*/, papPages, &fLockPgHack);
1562 if (rc == 0) {
1563 size_t iPage = cPages;
1564 while (iPage-- > 0)
1565 pReq->PgLst.aPages[iPage] = page_to_phys(papPages[iPage]);
1566 } else {
1567 cbRet = rc;
1568 break;
1569 }
1570
1571 /*
1572 * Issue the request and unlock the pages.
1573 */
1574 rc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile, cbChunk, cPages);
1575
1576 Assert(cPages <= cMaxPages);
1577 vbsf_unlock_user_pages(papPages, cPages, true /*fSetDirty*/, fLockPgHack);
1578
1579 if (RT_SUCCESS(rc)) {
1580 /*
1581 * Success, advance position and buffer.
1582 */
1583 uint32_t cbActual = pReq->Parms.cb32Read.u.value32;
1584 AssertStmt(cbActual <= cbChunk, cbActual = cbChunk);
1585 cbRet += cbActual;
1586 offFile += cbActual;
1587 buf = (uint8_t *)buf + cbActual;
1588 size -= cbActual;
1589
1590 /*
1591 * Are we done already? If so commit the new file offset.
1592 */
1593 if (!size || cbActual < cbChunk) {
1594 *off = offFile;
1595 break;
1596 }
1597 } else if (rc == VERR_NO_MEMORY && cMaxPages > 4) {
1598 /*
1599 * The host probably doesn't have enough heap to handle the
1600 * request, reduce the page count and retry.
1601 */
1602 cMaxPages /= 4;
1603 Assert(cMaxPages > 0);
1604 } else {
1605 /*
1606 * If we've successfully read stuff, return it rather than
1607 * the error. (Not sure if this is such a great idea...)
1608 */
1609 if (cbRet > 0) {
1610 SFLOGFLOW(("vbsf_reg_read: read at %#RX64 -> %Rrc; got cbRet=%#zx already\n", offFile, rc, cbRet));
1611 *off = offFile;
1612 } else {
1613 SFLOGFLOW(("vbsf_reg_read: read at %#RX64 -> %Rrc\n", offFile, rc));
1614 cbRet = -EPROTO;
1615 }
1616 break;
1617 }
1618 }
1619 }
1620 if (papPagesFree)
1621 kfree(papPages);
1622 if (pReq)
1623 VbglR0PhysHeapFree(pReq);
1624 SFLOGFLOW(("vbsf_reg_read: returns %zd (%#zx), *off=%RX64 [lock]\n", cbRet, cbRet, *off));
1625 return cbRet;
1626}
1627
1628
1629/**
1630 * Read from a regular file.
1631 *
1632 * @param file the file
1633 * @param buf the buffer
1634 * @param size length of the buffer
1635 * @param off offset within the file (in/out).
1636 * @returns the number of read bytes on success, Linux error code otherwise
1637 */
1638static ssize_t vbsf_reg_read(struct file *file, char /*__user*/ *buf, size_t size, loff_t *off)
1639{
1640 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
1641 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
1642 struct vbsf_reg_info *sf_r = file->private_data;
1643 struct address_space *mapping = inode->i_mapping;
1644
1645 SFLOGFLOW(("vbsf_reg_read: inode=%p file=%p buf=%p size=%#zx off=%#llx\n", inode, file, buf, size, *off));
1646
1647 if (!S_ISREG(inode->i_mode)) {
1648 LogFunc(("read from non regular file %d\n", inode->i_mode));
1649 return -EINVAL;
1650 }
1651
1652 /** @todo XXX Check read permission according to inode->i_mode! */
1653
1654 if (!size)
1655 return 0;
1656
1657 /*
1658 * If there is a mapping and O_DIRECT isn't in effect, we must at a
1659 * heed dirty pages in the mapping and read from them. For simplicity
1660 * though, we just do page cache reading when there are writable
1661 * mappings around with any kind of pages loaded.
1662 */
1663 if (vbsf_should_use_cached_read(file, mapping, pSuperInfo))
1664 return vbsf_reg_read_mapped(file, buf, size, off);
1665
1666 /*
1667 * For small requests, try use an embedded buffer provided we get a heap block
1668 * that does not cross page boundraries (see host code).
1669 */
1670 if (size <= PAGE_SIZE / 4 * 3 - RT_UOFFSETOF(VBOXSFREADEMBEDDEDREQ, abData[0]) /* see allocator */) {
1671 uint32_t const cbReq = RT_UOFFSETOF(VBOXSFREADEMBEDDEDREQ, abData[0]) + size;
1672 VBOXSFREADEMBEDDEDREQ *pReq = (VBOXSFREADEMBEDDEDREQ *)VbglR0PhysHeapAlloc(cbReq);
1673 if (pReq) {
1674 if ((PAGE_SIZE - ((uintptr_t)pReq & PAGE_OFFSET_MASK)) >= cbReq) {
1675 ssize_t cbRet;
1676 int vrc = VbglR0SfHostReqReadEmbedded(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, *off, (uint32_t)size);
1677 if (RT_SUCCESS(vrc)) {
1678 cbRet = pReq->Parms.cb32Read.u.value32;
1679 AssertStmt(cbRet <= (ssize_t)size, cbRet = size);
1680 if (copy_to_user(buf, pReq->abData, cbRet) == 0)
1681 *off += cbRet;
1682 else
1683 cbRet = -EFAULT;
1684 } else
1685 cbRet = -EPROTO;
1686 VbglR0PhysHeapFree(pReq);
1687 SFLOGFLOW(("vbsf_reg_read: returns %zd (%#zx), *off=%RX64 [embed]\n", cbRet, cbRet, *off));
1688 return cbRet;
1689 }
1690 VbglR0PhysHeapFree(pReq);
1691 }
1692 }
1693
1694#if 0 /* Turns out this is slightly slower than locking the pages even for 4KB reads (4.19/amd64). */
1695 /*
1696 * For medium sized requests try use a bounce buffer.
1697 */
1698 if (size <= _64K /** @todo make this configurable? */) {
1699 void *pvBounce = kmalloc(size, GFP_KERNEL);
1700 if (pvBounce) {
1701 VBOXSFREADPGLSTREQ *pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
1702 if (pReq) {
1703 ssize_t cbRet;
1704 int vrc = VbglR0SfHostReqReadContig(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, *off,
1705 (uint32_t)size, pvBounce, virt_to_phys(pvBounce));
1706 if (RT_SUCCESS(vrc)) {
1707 cbRet = pReq->Parms.cb32Read.u.value32;
1708 AssertStmt(cbRet <= (ssize_t)size, cbRet = size);
1709 if (copy_to_user(buf, pvBounce, cbRet) == 0)
1710 *off += cbRet;
1711 else
1712 cbRet = -EFAULT;
1713 } else
1714 cbRet = -EPROTO;
1715 VbglR0PhysHeapFree(pReq);
1716 kfree(pvBounce);
1717 SFLOGFLOW(("vbsf_reg_read: returns %zd (%#zx), *off=%RX64 [bounce]\n", cbRet, cbRet, *off));
1718 return cbRet;
1719 }
1720 kfree(pvBounce);
1721 }
1722 }
1723#endif
1724
1725 return vbsf_reg_read_locking(file, buf, size, off, pSuperInfo, sf_r);
1726}
1727
1728
1729/**
1730 * Helper the synchronizes the page cache content with something we just wrote
1731 * to the host.
1732 */
1733static void vbsf_reg_write_sync_page_cache(struct address_space *mapping, loff_t offFile, uint32_t cbRange,
1734 uint8_t const *pbSrcBuf, struct page **papSrcPages,
1735 uint32_t offSrcPage, size_t cSrcPages)
1736{
1737 Assert(offSrcPage < PAGE_SIZE);
1738 if (mapping && mapping->nrpages > 0) {
1739 /*
1740 * Work the pages in the write range.
1741 */
1742 while (cbRange > 0) {
1743 /*
1744 * Lookup the page at offFile. We're fine if there aren't
1745 * any there. We're skip if it's dirty or is being written
1746 * back, at least for now.
1747 */
1748 size_t const offDstPage = offFile & PAGE_OFFSET_MASK;
1749 size_t const cbToCopy = RT_MIN(PAGE_SIZE - offDstPage, cbRange);
1750 pgoff_t const idxPage = offFile >> PAGE_SHIFT;
1751 struct page *pDstPage = find_lock_page(mapping, idxPage);
1752 if (pDstPage) {
1753 if ( pDstPage->mapping == mapping /* ignore if re-purposed (paranoia) */
1754 && pDstPage->index == idxPage
1755 && !PageDirty(pDstPage) /* ignore if dirty */
1756 && !PageWriteback(pDstPage) /* ignore if being written back */ ) {
1757 /*
1758 * Map the page and do the copying.
1759 */
1760 uint8_t *pbDst = (uint8_t *)kmap(pDstPage);
1761 if (pbSrcBuf)
1762 memcpy(&pbDst[offDstPage], pbSrcBuf, cbToCopy);
1763 else {
1764 uint32_t const cbSrc0 = PAGE_SIZE - offSrcPage;
1765 uint8_t const *pbSrc = (uint8_t const *)kmap(papSrcPages[0]);
1766 AssertMsg(cSrcPages >= 1, ("offFile=%#llx cbRange=%#zx cbToCopy=%#zx\n", offFile, cbRange, cbToCopy));
1767 memcpy(&pbDst[offDstPage], &pbSrc[offSrcPage], RT_MIN(cbToCopy, cbSrc0));
1768 kunmap(papSrcPages[0]);
1769 if (cbToCopy > cbSrc0) {
1770 AssertMsg(cSrcPages >= 2, ("offFile=%#llx cbRange=%#zx cbToCopy=%#zx\n", offFile, cbRange, cbToCopy));
1771 pbSrc = (uint8_t const *)kmap(papSrcPages[1]);
1772 memcpy(&pbDst[offDstPage + cbSrc0], pbSrc, cbToCopy - cbSrc0);
1773 kunmap(papSrcPages[1]);
1774 }
1775 }
1776 kunmap(pDstPage);
1777 flush_dcache_page(pDstPage);
1778 if (cbToCopy == PAGE_SIZE)
1779 SetPageUptodate(pDstPage);
1780# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 10)
1781 mark_page_accessed(pDstPage);
1782# endif
1783 } else
1784 SFLOGFLOW(("vbsf_reg_write_sync_page_cache: Skipping page %p: mapping=%p (vs %p) writeback=%d offset=%#lx (vs%#lx)\n",
1785 pDstPage, pDstPage->mapping, mapping, PageWriteback(pDstPage), pDstPage->index, idxPage));
1786 unlock_page(pDstPage);
1787 vbsf_put_page(pDstPage);
1788 }
1789
1790 /*
1791 * Advance.
1792 */
1793 if (pbSrcBuf)
1794 pbSrcBuf += cbToCopy;
1795 else
1796 {
1797 offSrcPage += cbToCopy;
1798 Assert(offSrcPage < PAGE_SIZE * 2);
1799 if (offSrcPage >= PAGE_SIZE) {
1800 offSrcPage &= PAGE_OFFSET_MASK;
1801 papSrcPages++;
1802# ifdef VBOX_STRICT
1803 Assert(cSrcPages > 0);
1804 cSrcPages--;
1805# endif
1806 }
1807 }
1808 offFile += cbToCopy;
1809 cbRange -= cbToCopy;
1810 }
1811 }
1812 RT_NOREF(cSrcPages);
1813}
1814
1815
1816/**
1817 * Fallback case of vbsf_reg_write() that locks the user buffers and let the host
1818 * write directly to them.
1819 */
1820static ssize_t vbsf_reg_write_locking(struct file *file, const char /*__user*/ *buf, size_t size, loff_t *off, loff_t offFile,
1821 struct inode *inode, struct vbsf_inode_info *sf_i,
1822 struct vbsf_super_info *pSuperInfo, struct vbsf_reg_info *sf_r)
1823{
1824 /*
1825 * Lock pages and execute the write, taking care not to pass the host
1826 * more than it can handle in one go or more than we care to allocate
1827 * page arrays for. The latter limit is set at just short of 32KB due
1828 * to how the physical heap works.
1829 */
1830 struct page *apPagesStack[16];
1831 struct page **papPages = &apPagesStack[0];
1832 struct page **papPagesFree = NULL;
1833 VBOXSFWRITEPGLSTREQ *pReq;
1834 ssize_t cbRet = -ENOMEM;
1835 size_t cPages = (((uintptr_t)buf & PAGE_OFFSET_MASK) + size + PAGE_OFFSET_MASK) >> PAGE_SHIFT;
1836 size_t cMaxPages = RT_MIN(RT_MAX(pSuperInfo->cMaxIoPages, 1), cPages);
1837 bool fLockPgHack;
1838
1839 pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFWRITEPGLSTREQ, PgLst.aPages[cMaxPages]));
1840 while (!pReq && cMaxPages > 4) {
1841 cMaxPages /= 2;
1842 pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFWRITEPGLSTREQ, PgLst.aPages[cMaxPages]));
1843 }
1844 if (pReq && cMaxPages > RT_ELEMENTS(apPagesStack))
1845 papPagesFree = papPages = kmalloc(cMaxPages * sizeof(sizeof(papPages[0])), GFP_KERNEL);
1846 if (pReq && papPages) {
1847 cbRet = 0;
1848 for (;;) {
1849 /*
1850 * Figure out how much to process now and lock the user pages.
1851 */
1852 int rc;
1853 size_t cbChunk = (uintptr_t)buf & PAGE_OFFSET_MASK;
1854 pReq->PgLst.offFirstPage = (uint16_t)cbChunk;
1855 cPages = RT_ALIGN_Z(cbChunk + size, PAGE_SIZE) >> PAGE_SHIFT;
1856 if (cPages <= cMaxPages)
1857 cbChunk = size;
1858 else {
1859 cPages = cMaxPages;
1860 cbChunk = (cMaxPages << PAGE_SHIFT) - cbChunk;
1861 }
1862
1863 rc = vbsf_lock_user_pages((uintptr_t)buf, cPages, false /*fWrite*/, papPages, &fLockPgHack);
1864 if (rc == 0) {
1865 size_t iPage = cPages;
1866 while (iPage-- > 0)
1867 pReq->PgLst.aPages[iPage] = page_to_phys(papPages[iPage]);
1868 } else {
1869 cbRet = rc;
1870 break;
1871 }
1872
1873 /*
1874 * Issue the request and unlock the pages.
1875 */
1876 rc = VbglR0SfHostReqWritePgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile, cbChunk, cPages);
1877 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
1878 if (RT_SUCCESS(rc)) {
1879 /*
1880 * Success, advance position and buffer.
1881 */
1882 uint32_t cbActual = pReq->Parms.cb32Write.u.value32;
1883 AssertStmt(cbActual <= cbChunk, cbActual = cbChunk);
1884
1885 vbsf_reg_write_sync_page_cache(inode->i_mapping, offFile, cbActual, NULL /*pbKrnlBuf*/,
1886 papPages, (uintptr_t)buf & PAGE_OFFSET_MASK, cPages);
1887 Assert(cPages <= cMaxPages);
1888 vbsf_unlock_user_pages(papPages, cPages, false /*fSetDirty*/, fLockPgHack);
1889
1890 cbRet += cbActual;
1891 buf = (uint8_t *)buf + cbActual;
1892 size -= cbActual;
1893
1894 offFile += cbActual;
1895 if ((file->f_flags & O_APPEND) && (g_fSfFeatures & SHFL_FEATURE_WRITE_UPDATES_OFFSET))
1896 offFile = pReq->Parms.off64Write.u.value64;
1897 if (offFile > i_size_read(inode))
1898 i_size_write(inode, offFile);
1899
1900 sf_i->force_restat = 1; /* mtime (and size) may have changed */
1901
1902 /*
1903 * Are we done already? If so commit the new file offset.
1904 */
1905 if (!size || cbActual < cbChunk) {
1906 *off = offFile;
1907 break;
1908 }
1909 } else {
1910 vbsf_unlock_user_pages(papPages, cPages, false /*fSetDirty*/, fLockPgHack);
1911 if (rc == VERR_NO_MEMORY && cMaxPages > 4) {
1912 /*
1913 * The host probably doesn't have enough heap to handle the
1914 * request, reduce the page count and retry.
1915 */
1916 cMaxPages /= 4;
1917 Assert(cMaxPages > 0);
1918 } else {
1919 /*
1920 * If we've successfully written stuff, return it rather than
1921 * the error. (Not sure if this is such a great idea...)
1922 */
1923 if (cbRet > 0) {
1924 SFLOGFLOW(("vbsf_reg_write: write at %#RX64 -> %Rrc; got cbRet=%#zx already\n", offFile, rc, cbRet));
1925 *off = offFile;
1926 } else {
1927 SFLOGFLOW(("vbsf_reg_write: write at %#RX64 -> %Rrc\n", offFile, rc));
1928 cbRet = -EPROTO;
1929 }
1930 break;
1931 }
1932 }
1933 }
1934 }
1935 if (papPagesFree)
1936 kfree(papPages);
1937 if (pReq)
1938 VbglR0PhysHeapFree(pReq);
1939 SFLOGFLOW(("vbsf_reg_write: returns %zd (%#zx), *off=%RX64 [lock]\n", cbRet, cbRet, *off));
1940 return cbRet;
1941}
1942
1943
1944/**
1945 * Write to a regular file.
1946 *
1947 * @param file the file
1948 * @param buf the buffer
1949 * @param size length of the buffer
1950 * @param off offset within the file
1951 * @returns the number of written bytes on success, Linux error code otherwise
1952 */
1953static ssize_t vbsf_reg_write(struct file *file, const char *buf, size_t size, loff_t * off)
1954{
1955 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
1956 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
1957 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
1958 struct vbsf_reg_info *sf_r = file->private_data;
1959 struct address_space *mapping = inode->i_mapping;
1960 loff_t pos;
1961
1962 SFLOGFLOW(("vbsf_reg_write: inode=%p file=%p buf=%p size=%#zx off=%#llx\n", inode, file, buf, size, *off));
1963 Assert(sf_i);
1964 Assert(pSuperInfo);
1965 Assert(sf_r);
1966 AssertReturn(S_ISREG(inode->i_mode), -EINVAL);
1967
1968 pos = *off;
1969 if (file->f_flags & O_APPEND)
1970 pos = i_size_read(inode);
1971
1972 /** @todo XXX Check write permission according to inode->i_mode! */
1973
1974 if (!size) {
1975 if (file->f_flags & O_APPEND) /** @todo check if this is the consensus behavior... */
1976 *off = pos;
1977 return 0;
1978 }
1979
1980 /** @todo Implement the read-write caching mode. */
1981
1982 /*
1983 * If there are active writable mappings, coordinate with any
1984 * pending writes via those.
1985 */
1986 if ( mapping
1987 && mapping->nrpages > 0
1988 && mapping_writably_mapped(mapping)) {
1989#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32)
1990 int err = filemap_fdatawait_range(mapping, pos, pos + size - 1);
1991 if (err)
1992 return err;
1993#else
1994 /** @todo ... */
1995#endif
1996 }
1997
1998 /*
1999 * For small requests, try use an embedded buffer provided we get a heap block
2000 * that does not cross page boundraries (see host code).
2001 */
2002 if (size <= PAGE_SIZE / 4 * 3 - RT_UOFFSETOF(VBOXSFWRITEEMBEDDEDREQ, abData[0]) /* see allocator */) {
2003 uint32_t const cbReq = RT_UOFFSETOF(VBOXSFWRITEEMBEDDEDREQ, abData[0]) + size;
2004 VBOXSFWRITEEMBEDDEDREQ *pReq = (VBOXSFWRITEEMBEDDEDREQ *)VbglR0PhysHeapAlloc(cbReq);
2005 if ( pReq
2006 && (PAGE_SIZE - ((uintptr_t)pReq & PAGE_OFFSET_MASK)) >= cbReq) {
2007 ssize_t cbRet;
2008 if (copy_from_user(pReq->abData, buf, size) == 0) {
2009 int vrc = VbglR0SfHostReqWriteEmbedded(pSuperInfo->map.root, pReq, sf_r->Handle.hHost,
2010 pos, (uint32_t)size);
2011 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
2012 if (RT_SUCCESS(vrc)) {
2013 cbRet = pReq->Parms.cb32Write.u.value32;
2014 AssertStmt(cbRet <= (ssize_t)size, cbRet = size);
2015 vbsf_reg_write_sync_page_cache(mapping, pos, (uint32_t)cbRet, pReq->abData,
2016 NULL /*papSrcPages*/, 0 /*offSrcPage0*/, 0 /*cSrcPages*/);
2017 pos += cbRet;
2018 if ((file->f_flags & O_APPEND) && (g_fSfFeatures & SHFL_FEATURE_WRITE_UPDATES_OFFSET))
2019 pos = pReq->Parms.off64Write.u.value64;
2020 *off = pos;
2021 if (pos > i_size_read(inode))
2022 i_size_write(inode, pos);
2023 } else
2024 cbRet = -EPROTO;
2025 sf_i->force_restat = 1; /* mtime (and size) may have changed */
2026 } else
2027 cbRet = -EFAULT;
2028
2029 VbglR0PhysHeapFree(pReq);
2030 SFLOGFLOW(("vbsf_reg_write: returns %zd (%#zx), *off=%RX64 [embed]\n", cbRet, cbRet, *off));
2031 return cbRet;
2032 }
2033 if (pReq)
2034 VbglR0PhysHeapFree(pReq);
2035 }
2036
2037#if 0 /* Turns out this is slightly slower than locking the pages even for 4KB reads (4.19/amd64). */
2038 /*
2039 * For medium sized requests try use a bounce buffer.
2040 */
2041 if (size <= _64K /** @todo make this configurable? */) {
2042 void *pvBounce = kmalloc(size, GFP_KERNEL);
2043 if (pvBounce) {
2044 if (copy_from_user(pvBounce, buf, size) == 0) {
2045 VBOXSFWRITEPGLSTREQ *pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
2046 if (pReq) {
2047 ssize_t cbRet;
2048 int vrc = VbglR0SfHostReqWriteContig(pSuperInfo->map.root, pReq, sf_r->handle, pos,
2049 (uint32_t)size, pvBounce, virt_to_phys(pvBounce));
2050 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
2051 if (RT_SUCCESS(vrc)) {
2052 cbRet = pReq->Parms.cb32Write.u.value32;
2053 AssertStmt(cbRet <= (ssize_t)size, cbRet = size);
2054 vbsf_reg_write_sync_page_cache(mapping, pos, (uint32_t)cbRet, (uint8_t const *)pvBounce,
2055 NULL /*papSrcPages*/, 0 /*offSrcPage0*/, 0 /*cSrcPages*/);
2056 pos += cbRet;
2057 *off = pos;
2058 if (pos > i_size_read(inode))
2059 i_size_write(inode, pos);
2060 } else
2061 cbRet = -EPROTO;
2062 sf_i->force_restat = 1; /* mtime (and size) may have changed */
2063 VbglR0PhysHeapFree(pReq);
2064 kfree(pvBounce);
2065 SFLOGFLOW(("vbsf_reg_write: returns %zd (%#zx), *off=%RX64 [bounce]\n", cbRet, cbRet, *off));
2066 return cbRet;
2067 }
2068 kfree(pvBounce);
2069 } else {
2070 kfree(pvBounce);
2071 SFLOGFLOW(("vbsf_reg_write: returns -EFAULT, *off=%RX64 [bounce]\n", *off));
2072 return -EFAULT;
2073 }
2074 }
2075 }
2076#endif
2077
2078 return vbsf_reg_write_locking(file, buf, size, off, pos, inode, sf_i, pSuperInfo, sf_r);
2079}
2080
2081#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
2082
2083/**
2084 * Companion to vbsf_iter_lock_pages().
2085 */
2086DECLINLINE(void) vbsf_iter_unlock_pages(struct iov_iter *iter, struct page **papPages, size_t cPages, bool fSetDirty)
2087{
2088 /* We don't mark kernel pages dirty (KVECs, BVECs, PIPEs): */
2089 if (!iter_is_iovec(iter))
2090 fSetDirty = false;
2091
2092 while (cPages-- > 0)
2093 {
2094 struct page *pPage = papPages[cPages];
2095 if (fSetDirty && !PageReserved(pPage))
2096 set_page_dirty(pPage);
2097 vbsf_put_page(pPage);
2098 }
2099}
2100
2101
2102/**
2103 * Locks up to @a cMaxPages from the I/O vector iterator, advancing the
2104 * iterator.
2105 *
2106 * @returns 0 on success, negative errno value on failure.
2107 * @param iter The iterator to lock pages from.
2108 * @param fWrite Whether to write (true) or read (false) lock the pages.
2109 * @param pStash Where we stash peek results.
2110 * @param cMaxPages The maximum number of pages to get.
2111 * @param papPages Where to return the locked pages.
2112 * @param pcPages Where to return the number of pages.
2113 * @param poffPage0 Where to return the offset into the first page.
2114 * @param pcbChunk Where to return the number of bytes covered.
2115 */
2116static int vbsf_iter_lock_pages(struct iov_iter *iter, bool fWrite, struct vbsf_iter_stash *pStash, size_t cMaxPages,
2117 struct page **papPages, size_t *pcPages, size_t *poffPage0, size_t *pcbChunk)
2118{
2119 size_t cbChunk = 0;
2120 size_t cPages = 0;
2121 size_t offPage0 = 0;
2122 int rc = 0;
2123
2124 Assert(iov_iter_count(iter) + pStash->cb > 0);
2125 if (!(iter->type & ITER_KVEC)) {
2126 /*
2127 * Do we have a stashed page?
2128 */
2129 if (pStash->pPage) {
2130 papPages[0] = pStash->pPage;
2131 offPage0 = pStash->off;
2132 cbChunk = pStash->cb;
2133 cPages = 1;
2134 pStash->pPage = NULL;
2135 pStash->off = 0;
2136 pStash->cb = 0;
2137 if ( offPage0 + cbChunk < PAGE_SIZE
2138 || iov_iter_count(iter) == 0) {
2139 *poffPage0 = offPage0;
2140 *pcbChunk = cbChunk;
2141 *pcPages = cPages;
2142 SFLOGFLOW(("vbsf_iter_lock_pages: returns %d - cPages=%#zx offPage0=%#zx cbChunk=%zx (stashed)\n",
2143 rc, cPages, offPage0, cbChunk));
2144 return 0;
2145 }
2146 cMaxPages -= 1;
2147 SFLOG3(("vbsf_iter_lock_pages: Picked up stashed page: %#zx LB %#zx\n", offPage0, cbChunk));
2148 } else {
2149# if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
2150 /*
2151 * Copy out our starting point to assist rewinding.
2152 */
2153 pStash->offFromEnd = iov_iter_count(iter);
2154 pStash->Copy = *iter;
2155# endif
2156 }
2157
2158 /*
2159 * Get pages segment by segment.
2160 */
2161 do {
2162 /*
2163 * Make a special case of the first time thru here, since that's
2164 * the most typical scenario.
2165 */
2166 ssize_t cbSegRet;
2167 if (cPages == 0) {
2168# if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0)
2169 while (!iov_iter_single_seg_count(iter)) /* Old code didn't skip empty segments which caused EFAULTs. */
2170 iov_iter_advance(iter, 0);
2171# endif
2172 cbSegRet = iov_iter_get_pages(iter, papPages, iov_iter_count(iter), cMaxPages, &offPage0);
2173 if (cbSegRet > 0) {
2174 iov_iter_advance(iter, cbSegRet);
2175 cbChunk = (size_t)cbSegRet;
2176 cPages = RT_ALIGN_Z(offPage0 + cbSegRet, PAGE_SIZE) >> PAGE_SHIFT;
2177 cMaxPages -= cPages;
2178 SFLOG3(("vbsf_iter_lock_pages: iov_iter_get_pages -> %#zx @ %#zx; %#zx pages [first]\n", cbSegRet, offPage0, cPages));
2179 if ( cMaxPages == 0
2180 || ((offPage0 + (size_t)cbSegRet) & PAGE_OFFSET_MASK))
2181 break;
2182 } else {
2183 AssertStmt(cbSegRet < 0, cbSegRet = -EFAULT);
2184 rc = (int)cbSegRet;
2185 break;
2186 }
2187 } else {
2188 /*
2189 * Probe first page of new segment to check that we've got a zero offset and
2190 * can continue on the current chunk. Stash the page if the offset isn't zero.
2191 */
2192 size_t offPgProbe;
2193 size_t cbSeg = iov_iter_single_seg_count(iter);
2194 while (!cbSeg) {
2195 iov_iter_advance(iter, 0);
2196 cbSeg = iov_iter_single_seg_count(iter);
2197 }
2198 cbSegRet = iov_iter_get_pages(iter, &papPages[cPages], iov_iter_count(iter), 1, &offPgProbe);
2199 if (cbSegRet > 0) {
2200 iov_iter_advance(iter, cbSegRet); /** @todo maybe not do this if we stash the page? */
2201 Assert(offPgProbe + cbSegRet <= PAGE_SIZE);
2202 if (offPgProbe == 0) {
2203 cbChunk += cbSegRet;
2204 cPages += 1;
2205 cMaxPages -= 1;
2206 SFLOG3(("vbsf_iter_lock_pages: iov_iter_get_pages(1) -> %#zx @ %#zx\n", cbSegRet, offPgProbe));
2207 if ( cMaxPages == 0
2208 || cbSegRet != PAGE_SIZE)
2209 break;
2210
2211 /*
2212 * Get the rest of the segment (if anything remaining).
2213 */
2214 cbSeg -= cbSegRet;
2215 if (cbSeg > 0) {
2216 cbSegRet = iov_iter_get_pages(iter, &papPages[cPages], iov_iter_count(iter), cMaxPages, &offPgProbe);
2217 if (cbSegRet > 0) {
2218 size_t const cPgRet = RT_ALIGN_Z((size_t)cbSegRet, PAGE_SIZE) >> PAGE_SHIFT;
2219 Assert(offPgProbe == 0);
2220 iov_iter_advance(iter, cbSegRet);
2221 SFLOG3(("vbsf_iter_lock_pages: iov_iter_get_pages() -> %#zx; %#zx pages\n", cbSegRet, cPgRet));
2222 cPages += cPgRet;
2223 cMaxPages -= cPgRet;
2224 cbChunk += cbSegRet;
2225 if ( cMaxPages == 0
2226 || ((size_t)cbSegRet & PAGE_OFFSET_MASK))
2227 break;
2228 } else {
2229 AssertStmt(cbSegRet < 0, cbSegRet = -EFAULT);
2230 rc = (int)cbSegRet;
2231 break;
2232 }
2233 }
2234 } else {
2235 /* The segment didn't start at a page boundrary, so stash it for
2236 the next round: */
2237 SFLOGFLOW(("vbsf_iter_lock_pages: iov_iter_get_pages(1) -> %#zx @ %#zx; stashed\n", cbSegRet, offPgProbe));
2238 Assert(papPages[cPages]);
2239 pStash->pPage = papPages[cPages];
2240 pStash->off = offPgProbe;
2241 pStash->cb = cbSegRet;
2242 break;
2243 }
2244 } else {
2245 AssertStmt(cbSegRet < 0, cbSegRet = -EFAULT);
2246 rc = (int)cbSegRet;
2247 break;
2248 }
2249 }
2250 Assert(cMaxPages > 0);
2251 } while (iov_iter_count(iter) > 0);
2252
2253 } else {
2254 /*
2255 * The silly iov_iter_get_pages_alloc() function doesn't handle KVECs,
2256 * so everyone needs to do that by themselves.
2257 *
2258 * Note! Fixes here may apply to rtR0MemObjNativeLockKernel()
2259 * and vbsf_lock_user_pages_failed_check_kernel() as well.
2260 */
2261# if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
2262 pStash->offFromEnd = iov_iter_count(iter);
2263 pStash->Copy = *iter;
2264# endif
2265 do {
2266 uint8_t *pbBuf;
2267 size_t offStart;
2268 size_t cPgSeg;
2269
2270 size_t cbSeg = iov_iter_single_seg_count(iter);
2271 while (!cbSeg) {
2272 iov_iter_advance(iter, 0);
2273 cbSeg = iov_iter_single_seg_count(iter);
2274 }
2275
2276# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)
2277 pbBuf = iter->kvec->iov_base + iter->iov_offset;
2278# else
2279 pbBuf = iter->iov->iov_base + iter->iov_offset;
2280# endif
2281 offStart = (uintptr_t)pbBuf & PAGE_OFFSET_MASK;
2282 if (!cPages)
2283 offPage0 = offStart;
2284 else if (offStart)
2285 break;
2286
2287 cPgSeg = RT_ALIGN_Z(cbSeg, PAGE_SIZE) >> PAGE_SHIFT;
2288 if (cPgSeg > cMaxPages) {
2289 cPgSeg = cMaxPages;
2290 cbSeg = (cPgSeg << PAGE_SHIFT) - offStart;
2291 }
2292
2293 rc = vbsf_lock_kernel_pages(pbBuf, fWrite, cPgSeg, &papPages[cPages]);
2294 if (rc == 0) {
2295 iov_iter_advance(iter, cbSeg);
2296 cbChunk += cbSeg;
2297 cPages += cPgSeg;
2298 cMaxPages -= cPgSeg;
2299 if ( cMaxPages == 0
2300 || ((offStart + cbSeg) & PAGE_OFFSET_MASK) != 0)
2301 break;
2302 } else
2303 break;
2304 } while (iov_iter_count(iter) > 0);
2305 }
2306
2307 /*
2308 * Clean up if we failed; set return values.
2309 */
2310 if (rc == 0) {
2311 /* likely */
2312 } else {
2313 if (cPages > 0)
2314 vbsf_iter_unlock_pages(iter, papPages, cPages, false /*fSetDirty*/);
2315 offPage0 = cbChunk = cPages = 0;
2316 }
2317 *poffPage0 = offPage0;
2318 *pcbChunk = cbChunk;
2319 *pcPages = cPages;
2320 SFLOGFLOW(("vbsf_iter_lock_pages: returns %d - cPages=%#zx offPage0=%#zx cbChunk=%zx\n", rc, cPages, offPage0, cbChunk));
2321 return rc;
2322}
2323
2324
2325/**
2326 * Rewinds the I/O vector.
2327 */
2328static bool vbsf_iter_rewind(struct iov_iter *iter, struct vbsf_iter_stash *pStash, size_t cbToRewind, size_t cbChunk)
2329{
2330 size_t cbExtra;
2331 if (!pStash->pPage) {
2332 cbExtra = 0;
2333 } else {
2334 cbExtra = pStash->cb;
2335 vbsf_put_page(pStash->pPage);
2336 pStash->pPage = NULL;
2337 pStash->cb = 0;
2338 pStash->off = 0;
2339 }
2340
2341# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
2342 iov_iter_revert(iter, cbToRewind + cbExtra);
2343 return true;
2344# else
2345 /** @todo impl this */
2346 return false;
2347# endif
2348}
2349
2350
2351/**
2352 * Cleans up the page locking stash.
2353 */
2354DECLINLINE(void) vbsf_iter_cleanup_stash(struct iov_iter *iter, struct vbsf_iter_stash *pStash)
2355{
2356 if (pStash->pPage)
2357 vbsf_iter_rewind(iter, pStash, 0, 0);
2358}
2359
2360
2361/**
2362 * Calculates the longest span of pages we could transfer to the host in a
2363 * single request.
2364 *
2365 * @returns Page count, non-zero.
2366 * @param iter The I/O vector iterator to inspect.
2367 */
2368static size_t vbsf_iter_max_span_of_pages(struct iov_iter *iter)
2369{
2370 size_t cPages;
2371# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
2372 if (iter_is_iovec(iter) || (iter->type & ITER_KVEC)) {
2373#endif
2374 const struct iovec *pCurIov = iter->iov;
2375 size_t cLeft = iter->nr_segs;
2376 size_t cPagesSpan = 0;
2377
2378 /* iovect and kvec are identical, except for the __user tagging of iov_base. */
2379 AssertCompileMembersSameSizeAndOffset(struct iovec, iov_base, struct kvec, iov_base);
2380 AssertCompileMembersSameSizeAndOffset(struct iovec, iov_len, struct kvec, iov_len);
2381 AssertCompile(sizeof(struct iovec) == sizeof(struct kvec));
2382
2383 cPages = 1;
2384 AssertReturn(cLeft > 0, cPages);
2385
2386 /* Special case: segment offset. */
2387 if (iter->iov_offset > 0) {
2388 if (iter->iov_offset < pCurIov->iov_len) {
2389 size_t const cbSegLeft = pCurIov->iov_len - iter->iov_offset;
2390 size_t const offPage0 = ((uintptr_t)pCurIov->iov_base + iter->iov_offset) & PAGE_OFFSET_MASK;
2391 cPages = cPagesSpan = RT_ALIGN_Z(offPage0 + cbSegLeft, PAGE_SIZE) >> PAGE_SHIFT;
2392 if ((offPage0 + cbSegLeft) & PAGE_OFFSET_MASK)
2393 cPagesSpan = 0;
2394 }
2395 SFLOGFLOW(("vbsf_iter: seg[0]= %p LB %#zx\n", pCurIov->iov_base, pCurIov->iov_len));
2396 pCurIov++;
2397 cLeft--;
2398 }
2399
2400 /* Full segments. */
2401 while (cLeft-- > 0) {
2402 if (pCurIov->iov_len > 0) {
2403 size_t const offPage0 = (uintptr_t)pCurIov->iov_base & PAGE_OFFSET_MASK;
2404 if (offPage0 == 0) {
2405 if (!(pCurIov->iov_len & PAGE_OFFSET_MASK)) {
2406 cPagesSpan += pCurIov->iov_len >> PAGE_SHIFT;
2407 } else {
2408 cPagesSpan += RT_ALIGN_Z(pCurIov->iov_len, PAGE_SIZE) >> PAGE_SHIFT;
2409 if (cPagesSpan > cPages)
2410 cPages = cPagesSpan;
2411 cPagesSpan = 0;
2412 }
2413 } else {
2414 if (cPagesSpan > cPages)
2415 cPages = cPagesSpan;
2416 if (!((offPage0 + pCurIov->iov_len) & PAGE_OFFSET_MASK)) {
2417 cPagesSpan = pCurIov->iov_len >> PAGE_SHIFT;
2418 } else {
2419 cPagesSpan += RT_ALIGN_Z(offPage0 + pCurIov->iov_len, PAGE_SIZE) >> PAGE_SHIFT;
2420 if (cPagesSpan > cPages)
2421 cPages = cPagesSpan;
2422 cPagesSpan = 0;
2423 }
2424 }
2425 }
2426 SFLOGFLOW(("vbsf_iter: seg[%u]= %p LB %#zx\n", iter->nr_segs - cLeft, pCurIov->iov_base, pCurIov->iov_len));
2427 pCurIov++;
2428 }
2429 if (cPagesSpan > cPages)
2430 cPages = cPagesSpan;
2431# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
2432 } else {
2433 /* Won't bother with accurate counts for the next two types, just make
2434 some rough estimates (does pipes have segments?): */
2435 size_t cSegs = iter->type & ITER_BVEC ? RT_MAX(1, iter->nr_segs) : 1;
2436 cPages = (iov_iter_count(iter) + (PAGE_SIZE * 2 - 2) * cSegs) >> PAGE_SHIFT;
2437 }
2438# endif
2439 SFLOGFLOW(("vbsf_iter_max_span_of_pages: returns %#zx\n", cPages));
2440 return cPages;
2441}
2442
2443
2444/**
2445 * Worker for vbsf_reg_read_iter() that deals with larger reads using page
2446 * locking.
2447 */
2448static ssize_t vbsf_reg_read_iter_locking(struct kiocb *kio, struct iov_iter *iter, size_t cbToRead,
2449 struct vbsf_super_info *pSuperInfo, struct vbsf_reg_info *sf_r)
2450{
2451 /*
2452 * Estimate how many pages we may possible submit in a single request so
2453 * that we can allocate matching request buffer and page array.
2454 */
2455 struct page *apPagesStack[16];
2456 struct page **papPages = &apPagesStack[0];
2457 struct page **papPagesFree = NULL;
2458 VBOXSFREADPGLSTREQ *pReq;
2459 ssize_t cbRet = 0;
2460 size_t cMaxPages = vbsf_iter_max_span_of_pages(iter);
2461 cMaxPages = RT_MIN(RT_MAX(pSuperInfo->cMaxIoPages, 2), cMaxPages);
2462
2463 pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ, PgLst.aPages[cMaxPages]));
2464 while (!pReq && cMaxPages > 4) {
2465 cMaxPages /= 2;
2466 pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ, PgLst.aPages[cMaxPages]));
2467 }
2468 if (pReq && cMaxPages > RT_ELEMENTS(apPagesStack))
2469 papPagesFree = papPages = kmalloc(cMaxPages * sizeof(sizeof(papPages[0])), GFP_KERNEL);
2470 if (pReq && papPages) {
2471
2472 /*
2473 * The read loop.
2474 */
2475 struct vbsf_iter_stash Stash = VBSF_ITER_STASH_INITIALIZER;
2476 do {
2477 /*
2478 * Grab as many pages as we can. This means that if adjacent
2479 * segments both starts and ends at a page boundrary, we can
2480 * do them both in the same transfer from the host.
2481 */
2482 size_t cPages = 0;
2483 size_t cbChunk = 0;
2484 size_t offPage0 = 0;
2485 int rc = vbsf_iter_lock_pages(iter, true /*fWrite*/, &Stash, cMaxPages, papPages, &cPages, &offPage0, &cbChunk);
2486 if (rc == 0) {
2487 size_t iPage = cPages;
2488 while (iPage-- > 0)
2489 pReq->PgLst.aPages[iPage] = page_to_phys(papPages[iPage]);
2490 pReq->PgLst.offFirstPage = (uint16_t)offPage0;
2491 AssertStmt(cbChunk <= cbToRead, cbChunk = cbToRead);
2492 } else {
2493 cbRet = rc;
2494 break;
2495 }
2496
2497 /*
2498 * Issue the request and unlock the pages.
2499 */
2500 rc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, kio->ki_pos, cbChunk, cPages);
2501 SFLOGFLOW(("vbsf_reg_read_iter_locking: VbglR0SfHostReqReadPgLst -> %d (cbActual=%#x cbChunk=%#zx of %#zx cPages=%#zx offPage0=%#x\n",
2502 rc, pReq->Parms.cb32Read.u.value32, cbChunk, cbToRead, cPages, offPage0));
2503
2504 vbsf_iter_unlock_pages(iter, papPages, cPages, true /*fSetDirty*/);
2505
2506 if (RT_SUCCESS(rc)) {
2507 /*
2508 * Success, advance position and buffer.
2509 */
2510 uint32_t cbActual = pReq->Parms.cb32Read.u.value32;
2511 AssertStmt(cbActual <= cbChunk, cbActual = cbChunk);
2512 cbRet += cbActual;
2513 kio->ki_pos += cbActual;
2514 cbToRead -= cbActual;
2515
2516 /*
2517 * Are we done already?
2518 */
2519 if (!cbToRead)
2520 break;
2521 if (cbActual < cbChunk) { /* We ASSUME end-of-file here. */
2522 if (vbsf_iter_rewind(iter, &Stash, cbChunk - cbActual, cbActual))
2523 iov_iter_truncate(iter, 0);
2524 break;
2525 }
2526 } else {
2527 /*
2528 * Try rewind the iter structure.
2529 */
2530 bool const fRewindOkay = vbsf_iter_rewind(iter, &Stash, cbChunk, cbChunk);
2531 if (rc == VERR_NO_MEMORY && cMaxPages > 4 && fRewindOkay) {
2532 /*
2533 * The host probably doesn't have enough heap to handle the
2534 * request, reduce the page count and retry.
2535 */
2536 cMaxPages /= 4;
2537 Assert(cMaxPages > 0);
2538 } else {
2539 /*
2540 * If we've successfully read stuff, return it rather than
2541 * the error. (Not sure if this is such a great idea...)
2542 */
2543 if (cbRet <= 0)
2544 cbRet = -EPROTO;
2545 break;
2546 }
2547 }
2548 } while (cbToRead > 0);
2549
2550 vbsf_iter_cleanup_stash(iter, &Stash);
2551 }
2552 else
2553 cbRet = -ENOMEM;
2554 if (papPagesFree)
2555 kfree(papPages);
2556 if (pReq)
2557 VbglR0PhysHeapFree(pReq);
2558 SFLOGFLOW(("vbsf_reg_read_iter_locking: returns %#zx (%zd)\n", cbRet, cbRet));
2559 return cbRet;
2560}
2561
2562
2563/**
2564 * Read into I/O vector iterator.
2565 *
2566 * @returns Number of bytes read on success, negative errno on error.
2567 * @param kio The kernel I/O control block (or something like that).
2568 * @param iter The I/O vector iterator describing the buffer.
2569 */
2570# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
2571static ssize_t vbsf_reg_read_iter(struct kiocb *kio, struct iov_iter *iter)
2572# else
2573static ssize_t vbsf_reg_aio_read(struct kiocb *kio, const struct iovec *iov, unsigned long cSegs, loff_t offFile)
2574# endif
2575{
2576# if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
2577 struct vbsf_iov_iter fake_iter = VBSF_IOV_ITER_INITIALIZER(cSegs, iov, 0 /*write*/);
2578 struct vbsf_iov_iter *iter = &fake_iter;
2579# endif
2580 size_t cbToRead = iov_iter_count(iter);
2581 struct inode *inode = VBSF_GET_F_DENTRY(kio->ki_filp)->d_inode;
2582 struct address_space *mapping = inode->i_mapping;
2583
2584 struct vbsf_reg_info *sf_r = kio->ki_filp->private_data;
2585 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
2586
2587 SFLOGFLOW(("vbsf_reg_read_iter: inode=%p file=%p size=%#zx off=%#llx type=%#x\n",
2588 inode, kio->ki_filp, cbToRead, kio->ki_pos, iter->type));
2589 AssertReturn(S_ISREG(inode->i_mode), -EINVAL);
2590
2591 /*
2592 * Do we have anything at all to do here?
2593 */
2594 if (!cbToRead)
2595 return 0;
2596
2597 /*
2598 * If there is a mapping and O_DIRECT isn't in effect, we must at a
2599 * heed dirty pages in the mapping and read from them. For simplicity
2600 * though, we just do page cache reading when there are writable
2601 * mappings around with any kind of pages loaded.
2602 */
2603 if (vbsf_should_use_cached_read(kio->ki_filp, mapping, pSuperInfo)) {
2604# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
2605 return generic_file_read_iter(kio, iter);
2606# else
2607 return generic_file_aio_read(kio, iov, cSegs, offFile);
2608# endif
2609 }
2610
2611 /*
2612 * Now now we reject async I/O requests.
2613 */
2614 if (!is_sync_kiocb(kio)) {
2615 SFLOGFLOW(("vbsf_reg_read_iter: async I/O not yet supported\n")); /** @todo extend FsPerf with AIO tests. */
2616 return -EOPNOTSUPP;
2617 }
2618
2619 /*
2620 * For small requests, try use an embedded buffer provided we get a heap block
2621 * that does not cross page boundraries (see host code).
2622 */
2623 if (cbToRead <= PAGE_SIZE / 4 * 3 - RT_UOFFSETOF(VBOXSFREADEMBEDDEDREQ, abData[0]) /* see allocator */) {
2624 uint32_t const cbReq = RT_UOFFSETOF(VBOXSFREADEMBEDDEDREQ, abData[0]) + cbToRead;
2625 VBOXSFREADEMBEDDEDREQ *pReq = (VBOXSFREADEMBEDDEDREQ *)VbglR0PhysHeapAlloc(cbReq);
2626 if (pReq) {
2627 if ((PAGE_SIZE - ((uintptr_t)pReq & PAGE_OFFSET_MASK)) >= cbReq) {
2628 ssize_t cbRet;
2629 int vrc = VbglR0SfHostReqReadEmbedded(pSuperInfo->map.root, pReq, sf_r->Handle.hHost,
2630 kio->ki_pos, (uint32_t)cbToRead);
2631 if (RT_SUCCESS(vrc)) {
2632 cbRet = pReq->Parms.cb32Read.u.value32;
2633 AssertStmt(cbRet <= (ssize_t)cbToRead, cbRet = cbToRead);
2634 if (copy_to_iter(pReq->abData, cbRet, iter) == cbRet) {
2635 kio->ki_pos += cbRet;
2636 if (cbRet < cbToRead)
2637 iov_iter_truncate(iter, 0);
2638 } else
2639 cbRet = -EFAULT;
2640 } else
2641 cbRet = -EPROTO;
2642 VbglR0PhysHeapFree(pReq);
2643 SFLOGFLOW(("vbsf_reg_read_iter: returns %#zx (%zd)\n", cbRet, cbRet));
2644 return cbRet;
2645 }
2646 VbglR0PhysHeapFree(pReq);
2647 }
2648 }
2649
2650 /*
2651 * Otherwise do the page locking thing.
2652 */
2653 return vbsf_reg_read_iter_locking(kio, iter, cbToRead, pSuperInfo, sf_r);
2654}
2655
2656
2657/**
2658 * Worker for vbsf_reg_write_iter() that deals with larger writes using page
2659 * locking.
2660 */
2661static ssize_t vbsf_reg_write_iter_locking(struct kiocb *kio, struct iov_iter *iter, size_t cbToWrite, loff_t offFile,
2662 struct vbsf_super_info *pSuperInfo, struct vbsf_reg_info *sf_r, struct inode *inode,
2663 struct vbsf_inode_info *sf_i, struct address_space *mapping, bool fAppend)
2664{
2665 /*
2666 * Estimate how many pages we may possible submit in a single request so
2667 * that we can allocate matching request buffer and page array.
2668 */
2669 struct page *apPagesStack[16];
2670 struct page **papPages = &apPagesStack[0];
2671 struct page **papPagesFree = NULL;
2672 VBOXSFWRITEPGLSTREQ *pReq;
2673 ssize_t cbRet = 0;
2674 size_t cMaxPages = vbsf_iter_max_span_of_pages(iter);
2675 cMaxPages = RT_MIN(RT_MAX(pSuperInfo->cMaxIoPages, 2), cMaxPages);
2676
2677 pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFWRITEPGLSTREQ, PgLst.aPages[cMaxPages]));
2678 while (!pReq && cMaxPages > 4) {
2679 cMaxPages /= 2;
2680 pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFWRITEPGLSTREQ, PgLst.aPages[cMaxPages]));
2681 }
2682 if (pReq && cMaxPages > RT_ELEMENTS(apPagesStack))
2683 papPagesFree = papPages = kmalloc(cMaxPages * sizeof(sizeof(papPages[0])), GFP_KERNEL);
2684 if (pReq && papPages) {
2685
2686 /*
2687 * The write loop.
2688 */
2689 struct vbsf_iter_stash Stash = VBSF_ITER_STASH_INITIALIZER;
2690 do {
2691 /*
2692 * Grab as many pages as we can. This means that if adjacent
2693 * segments both starts and ends at a page boundrary, we can
2694 * do them both in the same transfer from the host.
2695 */
2696 size_t cPages = 0;
2697 size_t cbChunk = 0;
2698 size_t offPage0 = 0;
2699 int rc = vbsf_iter_lock_pages(iter, false /*fWrite*/, &Stash, cMaxPages, papPages, &cPages, &offPage0, &cbChunk);
2700 if (rc == 0) {
2701 size_t iPage = cPages;
2702 while (iPage-- > 0)
2703 pReq->PgLst.aPages[iPage] = page_to_phys(papPages[iPage]);
2704 pReq->PgLst.offFirstPage = (uint16_t)offPage0;
2705 AssertStmt(cbChunk <= cbToWrite, cbChunk = cbToWrite);
2706 } else {
2707 cbRet = rc;
2708 break;
2709 }
2710
2711 /*
2712 * Issue the request and unlock the pages.
2713 */
2714 rc = VbglR0SfHostReqWritePgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile, cbChunk, cPages);
2715 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
2716 SFLOGFLOW(("vbsf_reg_write_iter_locking: VbglR0SfHostReqWritePgLst -> %d (cbActual=%#x cbChunk=%#zx of %#zx cPages=%#zx offPage0=%#x\n",
2717 rc, pReq->Parms.cb32Write.u.value32, cbChunk, cbToWrite, cPages, offPage0));
2718 if (RT_SUCCESS(rc)) {
2719 /*
2720 * Success, advance position and buffer.
2721 */
2722 uint32_t cbActual = pReq->Parms.cb32Write.u.value32;
2723 AssertStmt(cbActual <= cbChunk, cbActual = cbChunk);
2724
2725 vbsf_reg_write_sync_page_cache(mapping, offFile, cbActual, NULL /*pbSrcBuf*/, papPages, offPage0, cPages);
2726 vbsf_iter_unlock_pages(iter, papPages, cPages, false /*fSetDirty*/);
2727
2728 cbRet += cbActual;
2729 cbToWrite -= cbActual;
2730
2731 offFile += cbActual;
2732 if (fAppend && (g_fSfFeatures & SHFL_FEATURE_WRITE_UPDATES_OFFSET))
2733 offFile = pReq->Parms.off64Write.u.value64;
2734 kio->ki_pos = offFile;
2735 if (offFile > i_size_read(inode))
2736 i_size_write(inode, offFile);
2737
2738 sf_i->force_restat = 1; /* mtime (and size) may have changed */
2739
2740 /*
2741 * Are we done already?
2742 */
2743 if (!cbToWrite)
2744 break;
2745 if (cbActual < cbChunk) { /* We ASSUME end-of-file here. */
2746 if (vbsf_iter_rewind(iter, &Stash, cbChunk - cbActual, cbActual))
2747 iov_iter_truncate(iter, 0);
2748 break;
2749 }
2750 } else {
2751 /*
2752 * Try rewind the iter structure.
2753 */
2754 bool fRewindOkay;
2755 vbsf_iter_unlock_pages(iter, papPages, cPages, false /*fSetDirty*/);
2756 fRewindOkay = vbsf_iter_rewind(iter, &Stash, cbChunk, cbChunk);
2757 if (rc == VERR_NO_MEMORY && cMaxPages > 4 && fRewindOkay) {
2758 /*
2759 * The host probably doesn't have enough heap to handle the
2760 * request, reduce the page count and retry.
2761 */
2762 cMaxPages /= 4;
2763 Assert(cMaxPages > 0);
2764 } else {
2765 /*
2766 * If we've successfully written stuff, return it rather than
2767 * the error. (Not sure if this is such a great idea...)
2768 */
2769 if (cbRet <= 0)
2770 cbRet = -EPROTO;
2771 break;
2772 }
2773 }
2774 } while (cbToWrite > 0);
2775
2776 vbsf_iter_cleanup_stash(iter, &Stash);
2777 }
2778 else
2779 cbRet = -ENOMEM;
2780 if (papPagesFree)
2781 kfree(papPages);
2782 if (pReq)
2783 VbglR0PhysHeapFree(pReq);
2784 SFLOGFLOW(("vbsf_reg_write_iter_locking: returns %#zx (%zd)\n", cbRet, cbRet));
2785 return cbRet;
2786}
2787
2788
2789/**
2790 * Write from I/O vector iterator.
2791 *
2792 * @returns Number of bytes written on success, negative errno on error.
2793 * @param kio The kernel I/O control block (or something like that).
2794 * @param iter The I/O vector iterator describing the buffer.
2795 */
2796# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
2797static ssize_t vbsf_reg_write_iter(struct kiocb *kio, struct iov_iter *iter)
2798# else
2799static ssize_t vbsf_reg_aio_write(struct kiocb *kio, const struct iovec *iov, unsigned long cSegs, loff_t offFile)
2800# endif
2801{
2802# if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
2803 struct vbsf_iov_iter fake_iter = VBSF_IOV_ITER_INITIALIZER(cSegs, iov, 1 /*write*/);
2804 struct vbsf_iov_iter *iter = &fake_iter;
2805# endif
2806 size_t cbToWrite = iov_iter_count(iter);
2807 struct inode *inode = VBSF_GET_F_DENTRY(kio->ki_filp)->d_inode;
2808 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
2809 struct address_space *mapping = inode->i_mapping;
2810
2811 struct vbsf_reg_info *sf_r = kio->ki_filp->private_data;
2812 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
2813# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
2814 loff_t offFile = kio->ki_pos;
2815# endif
2816# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
2817 bool const fAppend = RT_BOOL(kio->ki_flags & IOCB_APPEND);
2818# else
2819 bool const fAppend = RT_BOOL(kio->ki_filp->f_flags & O_APPEND);
2820# endif
2821
2822
2823 SFLOGFLOW(("vbsf_reg_write_iter: inode=%p file=%p size=%#zx off=%#llx type=%#x\n",
2824 inode, kio->ki_filp, cbToWrite, offFile, iter->type));
2825 AssertReturn(S_ISREG(inode->i_mode), -EINVAL);
2826
2827 /*
2828 * Enforce APPEND flag (more later).
2829 */
2830 if (fAppend)
2831 kio->ki_pos = offFile = i_size_read(inode);
2832
2833 /*
2834 * Do we have anything at all to do here?
2835 */
2836 if (!cbToWrite)
2837 return 0;
2838
2839 /** @todo Implement the read-write caching mode. */
2840
2841 /*
2842 * Now now we reject async I/O requests.
2843 */
2844 if (!is_sync_kiocb(kio)) {
2845 SFLOGFLOW(("vbsf_reg_write_iter: async I/O not yet supported\n")); /** @todo extend FsPerf with AIO tests. */
2846 return -EOPNOTSUPP;
2847 }
2848
2849 /*
2850 * If there are active writable mappings, coordinate with any
2851 * pending writes via those.
2852 */
2853 if ( mapping
2854 && mapping->nrpages > 0
2855 && mapping_writably_mapped(mapping)) {
2856# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32)
2857 int err = filemap_fdatawait_range(mapping, offFile, offFile + cbToWrite - 1);
2858 if (err)
2859 return err;
2860# else
2861 /** @todo ... */
2862# endif
2863 }
2864
2865 /*
2866 * For small requests, try use an embedded buffer provided we get a heap block
2867 * that does not cross page boundraries (see host code).
2868 */
2869 if (cbToWrite <= PAGE_SIZE / 4 * 3 - RT_UOFFSETOF(VBOXSFWRITEEMBEDDEDREQ, abData[0]) /* see allocator */) {
2870 uint32_t const cbReq = RT_UOFFSETOF(VBOXSFWRITEEMBEDDEDREQ, abData[0]) + cbToWrite;
2871 VBOXSFWRITEEMBEDDEDREQ *pReq = (VBOXSFWRITEEMBEDDEDREQ *)VbglR0PhysHeapAlloc(cbReq);
2872 if (pReq) {
2873 if ((PAGE_SIZE - ((uintptr_t)pReq & PAGE_OFFSET_MASK)) >= cbReq) {
2874 ssize_t cbRet;
2875 if (copy_from_iter(pReq->abData, cbToWrite, iter) == cbToWrite) {
2876 int vrc = VbglR0SfHostReqWriteEmbedded(pSuperInfo->map.root, pReq, sf_r->Handle.hHost,
2877 offFile, (uint32_t)cbToWrite);
2878 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
2879 if (RT_SUCCESS(vrc)) {
2880 cbRet = pReq->Parms.cb32Write.u.value32;
2881 AssertStmt(cbRet <= (ssize_t)cbToWrite, cbRet = cbToWrite);
2882 vbsf_reg_write_sync_page_cache(mapping, offFile, (uint32_t)cbRet, pReq->abData,
2883 NULL /*papSrcPages*/, 0 /*offSrcPage0*/, 0 /*cSrcPages*/);
2884
2885 offFile += cbRet;
2886 if (fAppend && (g_fSfFeatures & SHFL_FEATURE_WRITE_UPDATES_OFFSET))
2887 offFile = pReq->Parms.off64Write.u.value64;
2888 kio->ki_pos = offFile;
2889 if (offFile > i_size_read(inode))
2890 i_size_write(inode, offFile);
2891
2892# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
2893 if ((size_t)cbRet < cbToWrite)
2894 iov_iter_revert(iter, cbToWrite - cbRet);
2895# endif
2896 } else
2897 cbRet = -EPROTO;
2898 sf_i->force_restat = 1; /* mtime (and size) may have changed */
2899 } else
2900 cbRet = -EFAULT;
2901 VbglR0PhysHeapFree(pReq);
2902 SFLOGFLOW(("vbsf_reg_write_iter: returns %#zx (%zd)\n", cbRet, cbRet));
2903 return cbRet;
2904 }
2905 VbglR0PhysHeapFree(pReq);
2906 }
2907 }
2908
2909 /*
2910 * Otherwise do the page locking thing.
2911 */
2912 return vbsf_reg_write_iter_locking(kio, iter, cbToWrite, offFile, pSuperInfo, sf_r, inode, sf_i, mapping, fAppend);
2913}
2914
2915#endif /* >= 2.6.19 */
2916
2917/**
2918 * Used by vbsf_reg_open() and vbsf_inode_atomic_open() to
2919 *
2920 * @returns shared folders create flags.
2921 * @param fLnxOpen The linux O_XXX flags to convert.
2922 * @param pfHandle Pointer to vbsf_handle::fFlags.
2923 * @param pszCaller Caller, for logging purposes.
2924 */
2925uint32_t vbsf_linux_oflags_to_vbox(unsigned fLnxOpen, uint32_t *pfHandle, const char *pszCaller)
2926{
2927 uint32_t fVBoxFlags = SHFL_CF_ACCESS_DENYNONE;
2928
2929 /*
2930 * Disposition.
2931 */
2932 if (fLnxOpen & O_CREAT) {
2933 Log(("%s: O_CREAT set\n", pszCaller));
2934 fVBoxFlags |= SHFL_CF_ACT_CREATE_IF_NEW;
2935 if (fLnxOpen & O_EXCL) {
2936 Log(("%s: O_EXCL set\n", pszCaller));
2937 fVBoxFlags |= SHFL_CF_ACT_FAIL_IF_EXISTS;
2938 } else if (fLnxOpen & O_TRUNC) {
2939 Log(("%s: O_TRUNC set\n", pszCaller));
2940 fVBoxFlags |= SHFL_CF_ACT_OVERWRITE_IF_EXISTS;
2941 } else
2942 fVBoxFlags |= SHFL_CF_ACT_OPEN_IF_EXISTS;
2943 } else {
2944 fVBoxFlags |= SHFL_CF_ACT_FAIL_IF_NEW;
2945 if (fLnxOpen & O_TRUNC) {
2946 Log(("%s: O_TRUNC set\n", pszCaller));
2947 fVBoxFlags |= SHFL_CF_ACT_OVERWRITE_IF_EXISTS;
2948 }
2949 }
2950
2951 /*
2952 * Access.
2953 */
2954 switch (fLnxOpen & O_ACCMODE) {
2955 case O_RDONLY:
2956 fVBoxFlags |= SHFL_CF_ACCESS_READ;
2957 *pfHandle |= VBSF_HANDLE_F_READ;
2958 break;
2959
2960 case O_WRONLY:
2961 fVBoxFlags |= SHFL_CF_ACCESS_WRITE;
2962 *pfHandle |= VBSF_HANDLE_F_WRITE;
2963 break;
2964
2965 case O_RDWR:
2966 fVBoxFlags |= SHFL_CF_ACCESS_READWRITE;
2967 *pfHandle |= VBSF_HANDLE_F_READ | VBSF_HANDLE_F_WRITE;
2968 break;
2969
2970 default:
2971 BUG();
2972 }
2973
2974 if (fLnxOpen & O_APPEND) {
2975 Log(("%s: O_APPEND set\n", pszCaller));
2976 fVBoxFlags |= SHFL_CF_ACCESS_APPEND;
2977 *pfHandle |= VBSF_HANDLE_F_APPEND;
2978 }
2979
2980 /*
2981 * Only directories?
2982 */
2983 if (fLnxOpen & O_DIRECTORY) {
2984 Log(("%s: O_DIRECTORY set\n", pszCaller));
2985 fVBoxFlags |= SHFL_CF_DIRECTORY;
2986 }
2987
2988 return fVBoxFlags;
2989}
2990
2991
2992/**
2993 * Open a regular file.
2994 *
2995 * @param inode the inode
2996 * @param file the file
2997 * @returns 0 on success, Linux error code otherwise
2998 */
2999static int vbsf_reg_open(struct inode *inode, struct file *file)
3000{
3001 int rc, rc_linux = 0;
3002 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
3003 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
3004 struct dentry *dentry = VBSF_GET_F_DENTRY(file);
3005 struct vbsf_reg_info *sf_r;
3006 VBOXSFCREATEREQ *pReq;
3007
3008 SFLOGFLOW(("vbsf_reg_open: inode=%p file=%p flags=%#x %s\n", inode, file, file->f_flags, sf_i ? sf_i->path->String.ach : NULL));
3009 Assert(pSuperInfo);
3010 Assert(sf_i);
3011
3012 sf_r = kmalloc(sizeof(*sf_r), GFP_KERNEL);
3013 if (!sf_r) {
3014 LogRelFunc(("could not allocate reg info\n"));
3015 return -ENOMEM;
3016 }
3017
3018 RTListInit(&sf_r->Handle.Entry);
3019 sf_r->Handle.cRefs = 1;
3020 sf_r->Handle.fFlags = VBSF_HANDLE_F_FILE | VBSF_HANDLE_F_MAGIC;
3021 sf_r->Handle.hHost = SHFL_HANDLE_NIL;
3022
3023 /* Already open? */
3024 if (sf_i->handle != SHFL_HANDLE_NIL) {
3025 /*
3026 * This inode was created with vbsf_create_worker(). Check the CreateFlags:
3027 * O_CREAT, O_TRUNC: inherent true (file was just created). Not sure
3028 * about the access flags (SHFL_CF_ACCESS_*).
3029 */
3030 sf_i->force_restat = 1;
3031 sf_r->Handle.hHost = sf_i->handle;
3032 sf_i->handle = SHFL_HANDLE_NIL;
3033 file->private_data = sf_r;
3034
3035 sf_r->Handle.fFlags |= VBSF_HANDLE_F_READ | VBSF_HANDLE_F_WRITE; /** @todo fix */
3036 vbsf_handle_append(sf_i, &sf_r->Handle);
3037 SFLOGFLOW(("vbsf_reg_open: returns 0 (#1) - sf_i=%p hHost=%#llx\n", sf_i, sf_r->Handle.hHost));
3038 return 0;
3039 }
3040
3041 pReq = (VBOXSFCREATEREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq) + sf_i->path->u16Size);
3042 if (!pReq) {
3043 kfree(sf_r);
3044 LogRelFunc(("Failed to allocate a VBOXSFCREATEREQ buffer!\n"));
3045 return -ENOMEM;
3046 }
3047 memcpy(&pReq->StrPath, sf_i->path, SHFLSTRING_HEADER_SIZE + sf_i->path->u16Size);
3048 RT_ZERO(pReq->CreateParms);
3049 pReq->CreateParms.Handle = SHFL_HANDLE_NIL;
3050
3051 /* We check the value of pReq->CreateParms.Handle afterwards to
3052 * find out if the call succeeded or failed, as the API does not seem
3053 * to cleanly distinguish error and informational messages.
3054 *
3055 * Furthermore, we must set pReq->CreateParms.Handle to SHFL_HANDLE_NIL
3056 * to make the shared folders host service use our fMode parameter */
3057
3058 /* We ignore O_EXCL, as the Linux kernel seems to call create
3059 beforehand itself, so O_EXCL should always fail. */
3060 pReq->CreateParms.CreateFlags = vbsf_linux_oflags_to_vbox(file->f_flags & ~O_EXCL, &sf_r->Handle.fFlags, __FUNCTION__);
3061 pReq->CreateParms.Info.Attr.fMode = inode->i_mode;
3062 LogFunc(("vbsf_reg_open: calling VbglR0SfHostReqCreate, file %s, flags=%#x, %#x\n",
3063 sf_i->path->String.utf8, file->f_flags, pReq->CreateParms.CreateFlags));
3064 rc = VbglR0SfHostReqCreate(pSuperInfo->map.root, pReq);
3065 if (RT_FAILURE(rc)) {
3066 LogFunc(("VbglR0SfHostReqCreate failed flags=%d,%#x rc=%Rrc\n", file->f_flags, pReq->CreateParms.CreateFlags, rc));
3067 kfree(sf_r);
3068 VbglR0PhysHeapFree(pReq);
3069 return -RTErrConvertToErrno(rc);
3070 }
3071
3072 if (pReq->CreateParms.Handle != SHFL_HANDLE_NIL) {
3073 vbsf_dentry_chain_increase_ttl(dentry);
3074 vbsf_update_inode(inode, sf_i, &pReq->CreateParms.Info, pSuperInfo, false /*fInodeLocked*/, 0 /*fSetAttrs*/);
3075 rc_linux = 0;
3076 } else {
3077 switch (pReq->CreateParms.Result) {
3078 case SHFL_PATH_NOT_FOUND:
3079 vbsf_dentry_invalidate_ttl(dentry);
3080 rc_linux = -ENOENT;
3081 break;
3082 case SHFL_FILE_NOT_FOUND:
3083 vbsf_dentry_invalidate_ttl(dentry);
3084 /** @todo sf_dentry_increase_parent_ttl(file->f_dentry); if we can trust it. */
3085 rc_linux = -ENOENT;
3086 break;
3087 case SHFL_FILE_EXISTS:
3088 vbsf_dentry_chain_increase_ttl(dentry);
3089 vbsf_update_inode(inode, sf_i, &pReq->CreateParms.Info, pSuperInfo, false /*fInodeLocked*/, 0 /*fSetAttrs*/);
3090 rc_linux = -EEXIST;
3091 break;
3092 default:
3093 vbsf_dentry_chain_increase_parent_ttl(dentry);
3094 rc_linux = 0;
3095 break;
3096 }
3097 }
3098
3099 sf_r->Handle.hHost = pReq->CreateParms.Handle;
3100 file->private_data = sf_r;
3101 vbsf_handle_append(sf_i, &sf_r->Handle);
3102 VbglR0PhysHeapFree(pReq);
3103 SFLOGFLOW(("vbsf_reg_open: returns 0 (#2) - sf_i=%p hHost=%#llx\n", sf_i, sf_r->Handle.hHost));
3104 return rc_linux;
3105}
3106
3107
3108/**
3109 * Close a regular file.
3110 *
3111 * @param inode the inode
3112 * @param file the file
3113 * @returns 0 on success, Linux error code otherwise
3114 */
3115static int vbsf_reg_release(struct inode *inode, struct file *file)
3116{
3117 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
3118 struct vbsf_reg_info *sf_r = file->private_data;
3119
3120 SFLOGFLOW(("vbsf_reg_release: inode=%p file=%p\n", inode, file));
3121 if (sf_r) {
3122 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
3123 struct address_space *mapping = inode->i_mapping;
3124 Assert(pSuperInfo);
3125
3126 /* If we're closing the last handle for this inode, make sure the flush
3127 the mapping or we'll end up in vbsf_writepage without a handle. */
3128 if ( mapping
3129 && mapping->nrpages > 0
3130 /** @todo && last writable handle */ ) {
3131#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 25)
3132 if (filemap_fdatawrite(mapping) != -EIO)
3133#else
3134 if ( filemap_fdatasync(mapping) == 0
3135 && fsync_inode_data_buffers(inode) == 0)
3136#endif
3137 filemap_fdatawait(inode->i_mapping);
3138 }
3139
3140 /* Release sf_r, closing the handle if we're the last user. */
3141 file->private_data = NULL;
3142 vbsf_handle_release(&sf_r->Handle, pSuperInfo, "vbsf_reg_release");
3143
3144 sf_i->handle = SHFL_HANDLE_NIL;
3145 }
3146 return 0;
3147}
3148
3149
3150/**
3151 * Wrapper around generic/default seek function that ensures that we've got
3152 * the up-to-date file size when doing anything relative to EOF.
3153 *
3154 * The issue is that the host may extend the file while we weren't looking and
3155 * if the caller wishes to append data, it may end up overwriting existing data
3156 * if we operate with a stale size. So, we always retrieve the file size on EOF
3157 * relative seeks.
3158 */
3159static loff_t vbsf_reg_llseek(struct file *file, loff_t off, int whence)
3160{
3161 SFLOGFLOW(("vbsf_reg_llseek: file=%p off=%lld whence=%d\n", file, off, whence));
3162
3163 switch (whence) {
3164#ifdef SEEK_HOLE
3165 case SEEK_HOLE:
3166 case SEEK_DATA:
3167#endif
3168 case SEEK_END: {
3169 struct vbsf_reg_info *sf_r = file->private_data;
3170 int rc = vbsf_inode_revalidate_with_handle(VBSF_GET_F_DENTRY(file), sf_r->Handle.hHost,
3171 true /*fForce*/, false /*fInodeLocked*/);
3172 if (rc == 0)
3173 break;
3174 return rc;
3175 }
3176 }
3177
3178#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 8)
3179 return generic_file_llseek(file, off, whence);
3180#else
3181 return default_llseek(file, off, whence);
3182#endif
3183}
3184
3185
3186/**
3187 * Flush region of file - chiefly mmap/msync.
3188 *
3189 * We cannot use the noop_fsync / simple_sync_file here as that means
3190 * msync(,,MS_SYNC) will return before the data hits the host, thereby
3191 * causing coherency issues with O_DIRECT access to the same file as
3192 * well as any host interaction with the file.
3193 */
3194#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) \
3195 || (defined(CONFIG_SUSE_KERNEL) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 101) /** @todo figure when exactly */)
3196static int vbsf_reg_fsync(struct file *file, loff_t start, loff_t end, int datasync)
3197{
3198# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
3199 return __generic_file_fsync(file, start, end, datasync);
3200# else
3201 return generic_file_fsync(file, start, end, datasync);
3202# endif
3203}
3204#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35)
3205static int vbsf_reg_fsync(struct file *file, int datasync)
3206{
3207 return generic_file_fsync(file, datasync);
3208}
3209#else /* < 2.6.35 */
3210static int vbsf_reg_fsync(struct file *file, struct dentry *dentry, int datasync)
3211{
3212# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 31)
3213 return simple_fsync(file, dentry, datasync);
3214# else
3215 int rc;
3216 struct inode *inode = dentry->d_inode;
3217 AssertReturn(inode, -EINVAL);
3218
3219 /** @todo What about file_fsync()? (<= 2.5.11) */
3220
3221# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 12)
3222 rc = sync_mapping_buffers(inode->i_mapping);
3223 if ( rc == 0
3224 && (inode->i_state & I_DIRTY)
3225 && ((inode->i_state & I_DIRTY_DATASYNC) || !datasync)
3226 ) {
3227 struct writeback_control wbc = {
3228 .sync_mode = WB_SYNC_ALL,
3229 .nr_to_write = 0
3230 };
3231 rc = sync_inode(inode, &wbc);
3232 }
3233# else /* < 2.5.12 */
3234 /** @todo
3235 * Somethings is buggy here or in the 2.4.21-27.EL kernel I'm testing on.
3236 *
3237 * In theory we shouldn't need to do anything here, since msync will call
3238 * writepage() on each dirty page and we write them out synchronously. So, the
3239 * problem is elsewhere... Doesn't happen all the time either. Sigh.
3240 */
3241 rc = fsync_inode_buffers(inode);
3242# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 10)
3243 if (rc == 0 && datasync)
3244 rc = fsync_inode_data_buffers(inode);
3245# endif
3246
3247# endif /* < 2.5.12 */
3248 return rc;
3249# endif
3250}
3251#endif /* < 2.6.35 */
3252
3253
3254#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)
3255/**
3256 * Copy a datablock from one file to another on the host side.
3257 */
3258static ssize_t vbsf_reg_copy_file_range(struct file *pFileSrc, loff_t offSrc, struct file *pFileDst, loff_t offDst,
3259 size_t cbRange, unsigned int fFlags)
3260{
3261 ssize_t cbRet;
3262 if (g_uSfLastFunction >= SHFL_FN_COPY_FILE_PART) {
3263 struct inode *pInodeSrc = pFileSrc->f_inode;
3264 struct vbsf_inode_info *pInodeInfoSrc = VBSF_GET_INODE_INFO(pInodeSrc);
3265 struct vbsf_super_info *pSuperInfoSrc = VBSF_GET_SUPER_INFO(pInodeSrc->i_sb);
3266 struct vbsf_reg_info *pFileInfoSrc = (struct vbsf_reg_info *)pFileSrc->private_data;
3267 struct inode *pInodeDst = pInodeSrc;
3268 struct vbsf_inode_info *pInodeInfoDst = VBSF_GET_INODE_INFO(pInodeDst);
3269 struct vbsf_super_info *pSuperInfoDst = VBSF_GET_SUPER_INFO(pInodeDst->i_sb);
3270 struct vbsf_reg_info *pFileInfoDst = (struct vbsf_reg_info *)pFileDst->private_data;
3271 VBOXSFCOPYFILEPARTREQ *pReq;
3272
3273 /*
3274 * Some extra validation.
3275 */
3276 AssertPtrReturn(pInodeInfoSrc, -EOPNOTSUPP);
3277 Assert(pInodeInfoSrc->u32Magic == SF_INODE_INFO_MAGIC);
3278 AssertPtrReturn(pInodeInfoDst, -EOPNOTSUPP);
3279 Assert(pInodeInfoDst->u32Magic == SF_INODE_INFO_MAGIC);
3280
3281# if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
3282 if (!S_ISREG(pInodeSrc->i_mode) || !S_ISREG(pInodeDst->i_mode))
3283 return S_ISDIR(pInodeSrc->i_mode) || S_ISDIR(pInodeDst->i_mode) ? -EISDIR : -EINVAL;
3284# endif
3285
3286 /*
3287 * Allocate the request and issue it.
3288 */
3289 pReq = (VBOXSFCOPYFILEPARTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
3290 if (pReq) {
3291 int vrc = VbglR0SfHostReqCopyFilePart(pSuperInfoSrc->map.root, pFileInfoSrc->Handle.hHost, offSrc,
3292 pSuperInfoDst->map.root, pFileInfoDst->Handle.hHost, offDst,
3293 cbRange, 0 /*fFlags*/, pReq);
3294 if (RT_SUCCESS(vrc))
3295 cbRet = pReq->Parms.cb64ToCopy.u.value64;
3296 else if (vrc == VERR_NOT_IMPLEMENTED)
3297 cbRet = -EOPNOTSUPP;
3298 else
3299 cbRet = -RTErrConvertToErrno(vrc);
3300
3301 VbglR0PhysHeapFree(pReq);
3302 } else
3303 cbRet = -ENOMEM;
3304 } else {
3305 cbRet = -EOPNOTSUPP;
3306 }
3307 SFLOGFLOW(("vbsf_reg_copy_file_range: returns %zd\n", cbRet));
3308 return cbRet;
3309}
3310#endif /* > 4.5 */
3311
3312
3313#ifdef SFLOG_ENABLED
3314/*
3315 * This is just for logging page faults and such.
3316 */
3317
3318/** Pointer to the ops generic_file_mmap returns the first time it's called. */
3319static struct vm_operations_struct const *g_pGenericFileVmOps = NULL;
3320/** Merge of g_LoggingVmOpsTemplate and g_pGenericFileVmOps. */
3321static struct vm_operations_struct g_LoggingVmOps;
3322
3323
3324/* Generic page fault callback: */
3325# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
3326static vm_fault_t vbsf_vmlog_fault(struct vm_fault *vmf)
3327{
3328 vm_fault_t rc;
3329 SFLOGFLOW(("vbsf_vmlog_fault: vmf=%p flags=%#x addr=%p\n", vmf, vmf->flags, vmf->address));
3330 rc = g_pGenericFileVmOps->fault(vmf);
3331 SFLOGFLOW(("vbsf_vmlog_fault: returns %d\n", rc));
3332 return rc;
3333}
3334# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
3335static int vbsf_vmlog_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3336{
3337 int rc;
3338# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
3339 SFLOGFLOW(("vbsf_vmlog_fault: vma=%p vmf=%p flags=%#x addr=%p\n", vma, vmf, vmf->flags, vmf->address));
3340# else
3341 SFLOGFLOW(("vbsf_vmlog_fault: vma=%p vmf=%p flags=%#x addr=%p\n", vma, vmf, vmf->flags, vmf->virtual_address));
3342# endif
3343 rc = g_pGenericFileVmOps->fault(vma, vmf);
3344 SFLOGFLOW(("vbsf_vmlog_fault: returns %d\n", rc));
3345 return rc;
3346}
3347# endif
3348
3349
3350/* Special/generic page fault handler: */
3351# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
3352# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 1)
3353static struct page *vbsf_vmlog_nopage(struct vm_area_struct *vma, unsigned long address, int *type)
3354{
3355 struct page *page;
3356 SFLOGFLOW(("vbsf_vmlog_nopage: vma=%p address=%p type=%p:{%#x}\n", vma, address, type, type ? *type : 0));
3357 page = g_pGenericFileVmOps->nopage(vma, address, type);
3358 SFLOGFLOW(("vbsf_vmlog_nopage: returns %p\n", page));
3359 return page;
3360}
3361# else
3362static struct page *vbsf_vmlog_nopage(struct vm_area_struct *vma, unsigned long address, int write_access_or_unused)
3363{
3364 struct page *page;
3365 SFLOGFLOW(("vbsf_vmlog_nopage: vma=%p address=%p wau=%d\n", vma, address, write_access_or_unused));
3366 page = g_pGenericFileVmOps->nopage(vma, address, write_access_or_unused);
3367 SFLOGFLOW(("vbsf_vmlog_nopage: returns %p\n", page));
3368 return page;
3369}
3370# endif /* < 2.6.26 */
3371
3372
3373/* Special page fault callback for making something writable: */
3374# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
3375static vm_fault_t vbsf_vmlog_page_mkwrite(struct vm_fault *vmf)
3376{
3377 vm_fault_t rc;
3378 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: vmf=%p flags=%#x addr=%p\n", vmf, vmf->flags, vmf->address));
3379 rc = g_pGenericFileVmOps->page_mkwrite(vmf);
3380 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: returns %d\n", rc));
3381 return rc;
3382}
3383# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 30)
3384static int vbsf_vmlog_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
3385{
3386 int rc;
3387# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
3388 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: vma=%p vmf=%p flags=%#x addr=%p\n", vma, vmf, vmf->flags, vmf->address));
3389# else
3390 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: vma=%p vmf=%p flags=%#x addr=%p\n", vma, vmf, vmf->flags, vmf->virtual_address));
3391# endif
3392 rc = g_pGenericFileVmOps->page_mkwrite(vma, vmf);
3393 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: returns %d\n", rc));
3394 return rc;
3395}
3396# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
3397static int vbsf_vmlog_page_mkwrite(struct vm_area_struct *vma, struct page *page)
3398{
3399 int rc;
3400 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: vma=%p page=%p\n", vma, page));
3401 rc = g_pGenericFileVmOps->page_mkwrite(vma, page);
3402 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: returns %d\n", rc));
3403 return rc;
3404}
3405# endif
3406
3407
3408/* Special page fault callback for mapping pages: */
3409# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
3410static void vbsf_vmlog_map_pages(struct vm_fault *vmf, pgoff_t start, pgoff_t end)
3411{
3412 SFLOGFLOW(("vbsf_vmlog_map_pages: vmf=%p (flags=%#x addr=%p) start=%p end=%p\n", vmf, vmf->flags, vmf->address, start, end));
3413 g_pGenericFileVmOps->map_pages(vmf, start, end);
3414 SFLOGFLOW(("vbsf_vmlog_map_pages: returns\n"));
3415}
3416# elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)
3417static void vbsf_vmlog_map_pages(struct fault_env *fenv, pgoff_t start, pgoff_t end)
3418{
3419 SFLOGFLOW(("vbsf_vmlog_map_pages: fenv=%p (flags=%#x addr=%p) start=%p end=%p\n", fenv, fenv->flags, fenv->address, start, end));
3420 g_pGenericFileVmOps->map_pages(fenv, start, end);
3421 SFLOGFLOW(("vbsf_vmlog_map_pages: returns\n"));
3422}
3423# elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)
3424static void vbsf_vmlog_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf)
3425{
3426 SFLOGFLOW(("vbsf_vmlog_map_pages: vma=%p vmf=%p (flags=%#x addr=%p)\n", vma, vmf, vmf->flags, vmf->virtual_address));
3427 g_pGenericFileVmOps->map_pages(vma, vmf);
3428 SFLOGFLOW(("vbsf_vmlog_map_pages: returns\n"));
3429}
3430# endif
3431
3432
3433/** Overload template. */
3434static struct vm_operations_struct const g_LoggingVmOpsTemplate = {
3435# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
3436 .fault = vbsf_vmlog_fault,
3437# endif
3438# if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 25)
3439 .nopage = vbsf_vmlog_nopage,
3440# endif
3441# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
3442 .page_mkwrite = vbsf_vmlog_page_mkwrite,
3443# endif
3444# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)
3445 .map_pages = vbsf_vmlog_map_pages,
3446# endif
3447};
3448
3449/** file_operations::mmap wrapper for logging purposes. */
3450extern int vbsf_reg_mmap(struct file *file, struct vm_area_struct *vma)
3451{
3452 int rc;
3453 SFLOGFLOW(("vbsf_reg_mmap: file=%p vma=%p\n", file, vma));
3454 rc = generic_file_mmap(file, vma);
3455 if (rc == 0) {
3456 /* Merge the ops and template the first time thru (there's a race here). */
3457 if (g_pGenericFileVmOps == NULL) {
3458 uintptr_t const *puSrc1 = (uintptr_t *)vma->vm_ops;
3459 uintptr_t const *puSrc2 = (uintptr_t *)&g_LoggingVmOpsTemplate;
3460 uintptr_t volatile *puDst = (uintptr_t *)&g_LoggingVmOps;
3461 size_t cbLeft = sizeof(g_LoggingVmOps) / sizeof(*puDst);
3462 while (cbLeft-- > 0) {
3463 *puDst = *puSrc2 && *puSrc1 ? *puSrc2 : *puSrc1;
3464 puSrc1++;
3465 puSrc2++;
3466 puDst++;
3467 }
3468 g_pGenericFileVmOps = vma->vm_ops;
3469 vma->vm_ops = &g_LoggingVmOps;
3470 } else if (g_pGenericFileVmOps == vma->vm_ops)
3471 vma->vm_ops = &g_LoggingVmOps;
3472 else
3473 SFLOGFLOW(("vbsf_reg_mmap: Warning: vm_ops=%p, expected %p!\n", vma->vm_ops, g_pGenericFileVmOps));
3474 }
3475 SFLOGFLOW(("vbsf_reg_mmap: returns %d\n", rc));
3476 return rc;
3477}
3478
3479#endif /* SFLOG_ENABLED */
3480
3481
3482/**
3483 * File operations for regular files.
3484 *
3485 * Note on splice_read/splice_write/sendfile:
3486 * - Splice was introduced in 2.6.17. The generic_file_splice_read/write
3487 * methods go thru the page cache, which is undesirable and is why we
3488 * need to cook our own versions of the code as long as we cannot track
3489 * host-side writes and correctly invalidate the guest page-cache.
3490 * - Sendfile reimplemented using splice in 2.6.23.
3491 * - The default_file_splice_read/write no-page-cache fallback functions,
3492 * were introduced in 2.6.31. The write one work in page units.
3493 * - Since linux 3.16 there is iter_file_splice_write that uses iter_write.
3494 * - Since linux 4.9 the generic_file_splice_read function started using
3495 * read_iter.
3496 */
3497struct file_operations vbsf_reg_fops = {
3498 .open = vbsf_reg_open,
3499 .read = vbsf_reg_read,
3500 .write = vbsf_reg_write,
3501#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
3502 .read_iter = vbsf_reg_read_iter,
3503 .write_iter = vbsf_reg_write_iter,
3504#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
3505 .aio_read = vbsf_reg_aio_read,
3506 .aio_write = vbsf_reg_aio_write,
3507#endif
3508 .release = vbsf_reg_release,
3509#ifdef SFLOG_ENABLED
3510 .mmap = vbsf_reg_mmap,
3511#else
3512 .mmap = generic_file_mmap,
3513#endif
3514#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 17) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
3515 .splice_read = vbsf_splice_read,
3516#endif
3517#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
3518 .splice_write = iter_file_splice_write,
3519#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 17)
3520 .splice_write = vbsf_splice_write,
3521#endif
3522#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 30) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23)
3523 .sendfile = vbsf_reg_sendfile,
3524#endif
3525 .llseek = vbsf_reg_llseek,
3526 .fsync = vbsf_reg_fsync,
3527#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)
3528 .copy_file_range = vbsf_reg_copy_file_range,
3529#endif
3530};
3531
3532
3533/**
3534 * Inodes operations for regular files.
3535 */
3536struct inode_operations vbsf_reg_iops = {
3537#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 18)
3538 .getattr = vbsf_inode_getattr,
3539#else
3540 .revalidate = vbsf_inode_revalidate,
3541#endif
3542 .setattr = vbsf_inode_setattr,
3543};
3544
3545
3546
3547/*********************************************************************************************************************************
3548* Address Space Operations on Regular Files (for mmap, sendfile, direct I/O) *
3549*********************************************************************************************************************************/
3550
3551/**
3552 * Used to read the content of a page into the page cache.
3553 *
3554 * Needed for mmap and reads+writes when the file is mmapped in a
3555 * shared+writeable fashion.
3556 */
3557static int vbsf_readpage(struct file *file, struct page *page)
3558{
3559 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
3560 int err;
3561
3562 SFLOGFLOW(("vbsf_readpage: inode=%p file=%p page=%p off=%#llx\n", inode, file, page, (uint64_t)page->index << PAGE_SHIFT));
3563 Assert(PageLocked(page));
3564
3565 if (PageUptodate(page)) {
3566 unlock_page(page);
3567 return 0;
3568 }
3569
3570 if (!is_bad_inode(inode)) {
3571 VBOXSFREADPGLSTREQ *pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
3572 if (pReq) {
3573 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
3574 struct vbsf_reg_info *sf_r = file->private_data;
3575 uint32_t cbRead;
3576 int vrc;
3577
3578 pReq->PgLst.offFirstPage = 0;
3579 pReq->PgLst.aPages[0] = page_to_phys(page);
3580 vrc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root,
3581 pReq,
3582 sf_r->Handle.hHost,
3583 (uint64_t)page->index << PAGE_SHIFT,
3584 PAGE_SIZE,
3585 1 /*cPages*/);
3586
3587 cbRead = pReq->Parms.cb32Read.u.value32;
3588 AssertStmt(cbRead <= PAGE_SIZE, cbRead = PAGE_SIZE);
3589 VbglR0PhysHeapFree(pReq);
3590
3591 if (RT_SUCCESS(vrc)) {
3592 if (cbRead == PAGE_SIZE) {
3593 /* likely */
3594 } else {
3595 uint8_t *pbMapped = (uint8_t *)kmap(page);
3596 RT_BZERO(&pbMapped[cbRead], PAGE_SIZE - cbRead);
3597 kunmap(page);
3598 /** @todo truncate the inode file size? */
3599 }
3600
3601 flush_dcache_page(page);
3602 SetPageUptodate(page);
3603 unlock_page(page);
3604 return 0;
3605 }
3606 err = -RTErrConvertToErrno(vrc);
3607 } else
3608 err = -ENOMEM;
3609 } else
3610 err = -EIO;
3611 SetPageError(page);
3612 unlock_page(page);
3613 return err;
3614}
3615
3616
3617/**
3618 * Used to write out the content of a dirty page cache page to the host file.
3619 *
3620 * Needed for mmap and writes when the file is mmapped in a shared+writeable
3621 * fashion.
3622 */
3623#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 52)
3624static int vbsf_writepage(struct page *page, struct writeback_control *wbc)
3625#else
3626static int vbsf_writepage(struct page *page)
3627#endif
3628{
3629 struct address_space *mapping = page->mapping;
3630 struct inode *inode = mapping->host;
3631 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
3632 struct vbsf_handle *pHandle = vbsf_handle_find(sf_i, VBSF_HANDLE_F_WRITE, VBSF_HANDLE_F_APPEND);
3633 int err;
3634
3635 SFLOGFLOW(("vbsf_writepage: inode=%p page=%p off=%#llx pHandle=%p (%#llx)\n",
3636 inode, page, (uint64_t)page->index << PAGE_SHIFT, pHandle, pHandle ? pHandle->hHost : 0));
3637
3638 if (pHandle) {
3639 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
3640 VBOXSFWRITEPGLSTREQ *pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
3641 if (pReq) {
3642 uint64_t const cbFile = i_size_read(inode);
3643 uint64_t const offInFile = (uint64_t)page->index << PAGE_SHIFT;
3644 uint32_t const cbToWrite = page->index != (cbFile >> PAGE_SHIFT) ? PAGE_SIZE
3645 : (uint32_t)cbFile & (uint32_t)PAGE_OFFSET_MASK;
3646 int vrc;
3647
3648 pReq->PgLst.offFirstPage = 0;
3649 pReq->PgLst.aPages[0] = page_to_phys(page);
3650 vrc = VbglR0SfHostReqWritePgLst(pSuperInfo->map.root,
3651 pReq,
3652 pHandle->hHost,
3653 offInFile,
3654 cbToWrite,
3655 1 /*cPages*/);
3656 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
3657 AssertMsgStmt(pReq->Parms.cb32Write.u.value32 == cbToWrite || RT_FAILURE(vrc), /* lazy bird */
3658 ("%#x vs %#x\n", pReq->Parms.cb32Write, cbToWrite),
3659 vrc = VERR_WRITE_ERROR);
3660 VbglR0PhysHeapFree(pReq);
3661
3662 if (RT_SUCCESS(vrc)) {
3663 /* Update the inode if we've extended the file. */
3664 /** @todo is this necessary given the cbToWrite calc above? */
3665 uint64_t const offEndOfWrite = offInFile + cbToWrite;
3666 if ( offEndOfWrite > cbFile
3667 && offEndOfWrite > i_size_read(inode))
3668 i_size_write(inode, offEndOfWrite);
3669
3670 /* Update and unlock the page. */
3671 if (PageError(page))
3672 ClearPageError(page);
3673 SetPageUptodate(page);
3674 unlock_page(page);
3675
3676 vbsf_handle_release(pHandle, pSuperInfo, "vbsf_writepage");
3677 return 0;
3678 }
3679
3680 /*
3681 * We failed.
3682 */
3683 err = -EIO;
3684 } else
3685 err = -ENOMEM;
3686 vbsf_handle_release(pHandle, pSuperInfo, "vbsf_writepage");
3687 } else {
3688 /** @todo we could re-open the file here and deal with this... */
3689 static uint64_t volatile s_cCalls = 0;
3690 if (s_cCalls++ < 16)
3691 printk("vbsf_writepage: no writable handle for %s..\n", sf_i->path->String.ach);
3692 err = -EIO;
3693 }
3694 SetPageError(page);
3695 unlock_page(page);
3696 return err;
3697}
3698
3699
3700#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
3701/**
3702 * Called when writing thru the page cache (which we shouldn't be doing).
3703 */
3704int vbsf_write_begin(struct file *file, struct address_space *mapping, loff_t pos,
3705 unsigned len, unsigned flags, struct page **pagep, void **fsdata)
3706{
3707 /** @todo r=bird: We shouldn't ever get here, should we? Because we don't use
3708 * the page cache for any writes AFAIK. We could just as well use
3709 * simple_write_begin & simple_write_end here if we think we really
3710 * need to have non-NULL function pointers in the table... */
3711 static uint64_t volatile s_cCalls = 0;
3712 if (s_cCalls++ < 16) {
3713 printk("vboxsf: Unexpected call to vbsf_write_begin(pos=%#llx len=%#x flags=%#x)! Please report.\n",
3714 (unsigned long long)pos, len, flags);
3715 RTLogBackdoorPrintf("vboxsf: Unexpected call to vbsf_write_begin(pos=%#llx len=%#x flags=%#x)! Please report.\n",
3716 (unsigned long long)pos, len, flags);
3717# ifdef WARN_ON
3718 WARN_ON(1);
3719# endif
3720 }
3721 return simple_write_begin(file, mapping, pos, len, flags, pagep, fsdata);
3722}
3723#endif /* KERNEL_VERSION >= 2.6.24 */
3724
3725
3726#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 10)
3727
3728# ifdef VBOX_UEK
3729# undef iov_iter /* HACK ALERT! Don't put anything needing vbsf_iov_iter after this fun! */
3730# endif
3731
3732/**
3733 * This is needed to make open accept O_DIRECT as well as dealing with direct
3734 * I/O requests if we don't intercept them earlier.
3735 */
3736# if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) \
3737 || (defined(CONFIG_SUSE_KERNEL) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 73) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)) /** @todo Figure out when exactly. */
3738static ssize_t vbsf_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
3739# elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
3740static ssize_t vbsf_direct_IO(struct kiocb *iocb, struct iov_iter *iter, loff_t offset)
3741# elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) || defined(VBOX_UEK)
3742static ssize_t vbsf_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter, loff_t offset)
3743# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 6)
3744static ssize_t vbsf_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t offset, unsigned long nr_segs)
3745# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 55)
3746static int vbsf_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t offset, unsigned long nr_segs)
3747# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 41)
3748static int vbsf_direct_IO(int rw, struct file *file, const struct iovec *iov, loff_t offset, unsigned long nr_segs)
3749# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 35)
3750static int vbsf_direct_IO(int rw, struct inode *inode, const struct iovec *iov, loff_t offset, unsigned long nr_segs)
3751# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 26)
3752static int vbsf_direct_IO(int rw, struct inode *inode, char *buf, loff_t offset, size_t count)
3753# elif LINUX_VERSION_CODE == KERNEL_VERSION(2, 4, 21) && defined(I_NEW) /* RHEL3 Frankenkernel. */
3754static int vbsf_direct_IO(int rw, struct file *file, struct kiobuf *buf, unsigned long whatever1, int whatever2)
3755# else
3756static int vbsf_direct_IO(int rw, struct inode *inode, struct kiobuf *buf, unsigned long whatever1, int whatever2)
3757# endif
3758{
3759 TRACE();
3760 return -EINVAL;
3761}
3762
3763#endif
3764
3765/**
3766 * Address space (for the page cache) operations for regular files.
3767 *
3768 * @todo the FsPerf touch/flush (mmap) test fails on 4.4.0 (ubuntu 16.04 lts).
3769 */
3770struct address_space_operations vbsf_reg_aops = {
3771 .readpage = vbsf_readpage,
3772 .writepage = vbsf_writepage,
3773 /** @todo Need .writepages if we want msync performance... */
3774#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 12)
3775 .set_page_dirty = __set_page_dirty_buffers,
3776#endif
3777#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
3778 .write_begin = vbsf_write_begin,
3779 .write_end = simple_write_end,
3780#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 45)
3781 .prepare_write = simple_prepare_write,
3782 .commit_write = simple_commit_write,
3783#endif
3784#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 10)
3785 .direct_IO = vbsf_direct_IO,
3786#endif
3787};
3788
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette