The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=49587b8fb4d12166d30efb7111d32f4606a1e73b
commit 49587b8fb4d12166d30efb7111d32f4606a1e73b Author: Konstantin Belousov <[email protected]> AuthorDate: 2022-09-08 20:15:32 +0000 Commit: Konstantin Belousov <[email protected]> CommitDate: 2022-09-14 15:46:32 +0000 i386 copyout/in_fast: handle page fault from KVA access by delegating the work to the slow path. Some kernel memory, like pipe buffers, is pageable. We must not enable interrupts, and consequently, preemption, while in critical section in the fast copyout path, because we use pcpu buffers. If page fault occurs while copying from the pcpu copyout_buf to kernel memory, abort fast path and delegate work to the slow implementation. In collaboration with: pho, tijl Sponsored by: The FreeBSD Foundation MFC after: 1 week --- sys/i386/i386/copyout_fast.s | 21 +++++++++------------ sys/i386/i386/exception.s | 25 +++++++++++++++---------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/sys/i386/i386/copyout_fast.s b/sys/i386/i386/copyout_fast.s index 70647fe7613d..4391f36b18e4 100644 --- a/sys/i386/i386/copyout_fast.s +++ b/sys/i386/i386/copyout_fast.s @@ -47,32 +47,30 @@ ENTRY(copyout_fast) pushl %ebx movl 20(%ebp),%ebx /* KCR3 */ - movl PCPU(CURPCB),%edx - movl PCB_CR3(%edx),%edx /* UCR3 */ /* bcopy(%esi = kaddr, %edi = PCPU(copyout_buf), %ecx = len) */ movl 16(%ebp),%ecx movl 8(%ebp),%esi + movl %esp,%eax + movl $copyout_fault,%edx + cli movl PCPU(COPYOUT_BUF),%edi - rep; movsb +pf_y1: rep; movsb movl 16(%ebp),%ecx /* len */ movl PCPU(COPYOUT_BUF),%esi /* kaddr */ movl 12(%ebp),%edi /* uaddr */ - - movl %esp,%eax movl PCPU(TRAMPSTK),%esp - + movl PCPU(CURPCB),%edx + movl PCB_CR3(%edx),%edx /* UCR3 */ movl %edx,%cr3 movl $copyout_fault,%edx - /* bcopy(%esi = PCPU(copyout_buf), %edi = udaddr, %ecx = len) */ pf_x1: rep; movsb movl %ebx,%cr3 movl %eax,%esp sti - xorl %eax,%eax popl %ebx popl %edi @@ -93,10 +91,10 @@ ENTRY(copyin_fast) movl PCB_CR3(%eax),%edx /* UCR3 */ movl 16(%ebp),%ecx /* len */ movl 8(%ebp),%esi /* udaddr */ + movl %esp,%eax + cli movl PCPU(COPYOUT_BUF),%edi /* kaddr */ - - movl %esp,%eax movl PCPU(TRAMPSTK),%esp movl %edx,%cr3 movl $copyout_fault,%edx @@ -110,10 +108,9 @@ pf_x2: rep; movsb movl 16(%ebp),%ecx movl 12(%ebp),%edi movl PCPU(COPYOUT_BUF),%esi - rep; movsb +pf_y2: rep; movsb sti - xorl %eax,%eax popl %ebx popl %edi diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s index f8d61c884a21..e83d95e6afe1 100644 --- a/sys/i386/i386/exception.s +++ b/sys/i386/i386/exception.s @@ -130,15 +130,15 @@ IDTVEC(prot) jmp irettraps IDTVEC(page) testl $PSL_VM, TF_EFLAGS-TF_ERR(%esp) - jnz 4f + jnz 5f testb $SEL_RPL_MASK, TF_CS-TF_ERR(%esp) - jnz 4f + jnz 5f cmpl $PMAP_TRM_MIN_ADDRESS, TF_EIP-TF_ERR(%esp) - jb 4f + jb 5f pushl %eax movl TF_EIP-TF_ERR+4(%esp), %eax addl $1f, %eax - call 5f + call 6f 1: cmpl $pf_x1, %eax je 2f cmpl $pf_x2, %eax @@ -154,16 +154,21 @@ IDTVEC(page) cmpl $pf_x7, %eax je 2f cmpl $pf_x8, %eax - jne 3f -2: popl %eax - movl %ebx, %cr3 + je 2f + cmpl $pf_y1, %eax + je 3f + cmpl $pf_y2, %eax + je 3f + jmp 4f +2: movl %ebx, %cr3 +3: popl %eax movl %edx, TF_EIP-TF_ERR(%esp) addl $4, %esp iret -3: popl %eax -4: pushl $T_PAGEFLT +4: popl %eax +5: pushl $T_PAGEFLT jmp alltraps -5: subl (%esp), %eax +6: subl (%esp), %eax retl IDTVEC(rsvd_pti) IDTVEC(rsvd)
