Author: skra
Date: Tue Nov 24 13:57:41 2015
New Revision: 291258
URL: https://svnweb.freebsd.org/changeset/base/291258
Log:
Flush all kernel mappings from TLB(s) in time when they are cleared.
Replace tlb_flush_local() by tlb_flush() as even not global mappings
could be fetched to TLB(s) on other cores by speculative table walk.
From OS point of view, it was not a problem as either such mappings
were not used anymore or they were flushed from TLB(s) when reused.
However, from hardware point of view, it was a problem. Not flushed
mappings could be a target for speculative reads or prefetches (which
might be quite aggresive on ARM cores). As speculative read can fill
cacheline, it can cause a real problem, when physical page is reused,
but mapped with different memory attributes.
Anyhow, it's good to have only valid mappings in TLB(s).
Approved by: kib (mentor)
Modified:
head/sys/arm/arm/pmap-v6-new.c
head/sys/arm/include/sf_buf.h
Modified: head/sys/arm/arm/pmap-v6-new.c
==============================================================================
--- head/sys/arm/arm/pmap-v6-new.c Tue Nov 24 13:47:21 2015
(r291257)
+++ head/sys/arm/arm/pmap-v6-new.c Tue Nov 24 13:57:41 2015
(r291258)
@@ -1541,12 +1541,12 @@ pmap_pt2pg_zero(vm_page_t m)
panic("%s: CMAP2 busy", __func__);
pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(pa, PTE2_AP_KRW,
m->md.pat_mode));
- tlb_flush_local((vm_offset_t)sysmaps->CADDR2);
/* Even VM_ALLOC_ZERO request is only advisory. */
if ((m->flags & PG_ZERO) == 0)
pagezero(sysmaps->CADDR2);
pte2_sync_range((pt2_entry_t *)sysmaps->CADDR2, PAGE_SIZE);
pte2_clear(sysmaps->CMAP2);
+ tlb_flush((vm_offset_t)sysmaps->CADDR2);
sched_unpin();
mtx_unlock(&sysmaps->lock);
@@ -5470,7 +5470,6 @@ pmap_page_set_memattr(vm_page_t m, vm_me
struct sysmaps *sysmaps;
vm_memattr_t oma;
vm_paddr_t pa;
- vm_offset_t va;
oma = m->md.pat_mode;
m->md.pat_mode = ma;
@@ -5502,10 +5501,9 @@ pmap_page_set_memattr(vm_page_t m, vm_me
if (*sysmaps->CMAP2)
panic("%s: CMAP2 busy", __func__);
pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(pa, PTE2_AP_KRW, ma));
- va = (vm_offset_t)sysmaps->CADDR2;
- tlb_flush_local(va);
- dcache_wbinv_poc(va, pa, PAGE_SIZE);
+ dcache_wbinv_poc((vm_offset_t)sysmaps->CADDR2, pa, PAGE_SIZE);
pte2_clear(sysmaps->CMAP2);
+ tlb_flush((vm_offset_t)sysmaps->CADDR2);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -5594,9 +5592,9 @@ pmap_zero_page(vm_page_t m)
panic("%s: CMAP2 busy", __func__);
pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW,
m->md.pat_mode));
- tlb_flush_local((vm_offset_t)sysmaps->CADDR2);
pagezero(sysmaps->CADDR2);
pte2_clear(sysmaps->CMAP2);
+ tlb_flush((vm_offset_t)sysmaps->CADDR2);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -5619,12 +5617,12 @@ pmap_zero_page_area(vm_page_t m, int off
panic("%s: CMAP2 busy", __func__);
pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW,
m->md.pat_mode));
- tlb_flush_local((vm_offset_t)sysmaps->CADDR2);
if (off == 0 && size == PAGE_SIZE)
pagezero(sysmaps->CADDR2);
else
bzero(sysmaps->CADDR2 + off, size);
pte2_clear(sysmaps->CMAP2);
+ tlb_flush((vm_offset_t)sysmaps->CADDR2);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -5644,9 +5642,9 @@ pmap_zero_page_idle(vm_page_t m)
sched_pin();
pte2_store(CMAP3, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW,
m->md.pat_mode));
- tlb_flush_local((vm_offset_t)CADDR3);
pagezero(CADDR3);
pte2_clear(CMAP3);
+ tlb_flush((vm_offset_t)CADDR3);
sched_unpin();
}
@@ -5670,13 +5668,13 @@ pmap_copy_page(vm_page_t src, vm_page_t
panic("%s: CMAP2 busy", __func__);
pte2_store(sysmaps->CMAP1, PTE2_KERN_NG(VM_PAGE_TO_PHYS(src),
PTE2_AP_KR | PTE2_NM, src->md.pat_mode));
- tlb_flush_local((vm_offset_t)sysmaps->CADDR1);
pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(dst),
PTE2_AP_KRW, dst->md.pat_mode));
- tlb_flush_local((vm_offset_t)sysmaps->CADDR2);
bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE);
pte2_clear(sysmaps->CMAP1);
+ tlb_flush((vm_offset_t)sysmaps->CADDR1);
pte2_clear(sysmaps->CMAP2);
+ tlb_flush((vm_offset_t)sysmaps->CADDR2);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -5721,7 +5719,9 @@ pmap_copy_pages(vm_page_t ma[], vm_offse
xfersize -= cnt;
}
pte2_clear(sysmaps->CMAP1);
+ tlb_flush((vm_offset_t)sysmaps->CADDR1);
pte2_clear(sysmaps->CMAP2);
+ tlb_flush((vm_offset_t)sysmaps->CADDR2);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -5740,8 +5740,6 @@ pmap_quick_enter_page(vm_page_t m)
pte2_store(pte2p, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW,
pmap_page_get_memattr(m)));
- tlb_flush_local(qmap_addr);
-
return (qmap_addr);
}
@@ -5758,6 +5756,7 @@ pmap_quick_remove_page(vm_offset_t addr)
KASSERT(pte2_load(pte2p) != 0, ("%s: PTE2 not in use", __func__));
pte2_clear(pte2p);
+ tlb_flush(qmap_addr);
critical_exit();
}
@@ -6059,9 +6058,9 @@ pmap_dcache_wb_pou(vm_paddr_t pa, vm_siz
if (*sysmaps->CMAP3)
panic("%s: CMAP3 busy", __func__);
pte2_store(sysmaps->CMAP3, PTE2_KERN_NG(pa, PTE2_AP_KRW, ma));
- tlb_flush_local((vm_offset_t)sysmaps->CADDR3);
dcache_wb_pou((vm_offset_t)sysmaps->CADDR3 + (pa & PAGE_MASK), size);
pte2_clear(sysmaps->CMAP3);
+ tlb_flush((vm_offset_t)sysmaps->CADDR3);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
@@ -6313,13 +6312,13 @@ pmap_zero_page_check(vm_page_t m)
panic("%s: CMAP2 busy", __func__);
pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW,
m->md.pat_mode));
- tlb_flush_local((vm_offset_t)sysmaps->CADDR2);
end = (uint32_t*)(sysmaps->CADDR2 + PAGE_SIZE);
for (p = (uint32_t*)sysmaps->CADDR2; p < end; p++)
if (*p != 0)
panic("%s: page %p not zero, va: %p", __func__, m,
sysmaps->CADDR2);
pte2_clear(sysmaps->CMAP2);
+ tlb_flush((vm_offset_t)sysmaps->CADDR2);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}
Modified: head/sys/arm/include/sf_buf.h
==============================================================================
--- head/sys/arm/include/sf_buf.h Tue Nov 24 13:47:21 2015
(r291257)
+++ head/sys/arm/include/sf_buf.h Tue Nov 24 13:57:41 2015
(r291258)
@@ -44,7 +44,11 @@ static inline int
sf_buf_unmap(struct sf_buf *sf)
{
+#ifdef ARM_NEW_PMAP
+ pmap_qremove(sf->kva, 1);
+#else
pmap_kremove(sf->kva);
+#endif
return (1);
}
#endif /* !_MACHINE_SF_BUF_H_ */
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
