Extend mapping/unmapping operations to consider cache partitions specified via "coloring" and perform virt2phys mapping accordingly.
Also provide colored flushing of a memory region as well as an initialization helper to read the configuration parameters (way_size, root_map_offset). The root_map_offset is used as offset scratchpad to simplify the remapping_to/unmap_from root operations when loading and starting inmates. Refer to the Documentation/cache-coloring.md for an introduction of the coloring isolation software techniques. Signed-off-by: Andrea Bastoni <andrea.bast...@tum.de> --- .../arch/arm-common/include/asm/coloring.h | 45 +++++ .../arch/arm-common/include/asm/dcaches.h | 8 + hypervisor/arch/arm-common/mmu_cell.c | 68 +++++--- hypervisor/arch/arm64/Kbuild | 1 + hypervisor/arch/arm64/coloring.c | 108 ++++++++++++ hypervisor/arch/arm64/include/asm/coloring.h | 155 ++++++++++++++++++ hypervisor/arch/arm64/setup.c | 3 + hypervisor/control.c | 23 ++- hypervisor/include/jailhouse/control.h | 4 + include/jailhouse/coloring.h | 39 +++++ 10 files changed, 431 insertions(+), 23 deletions(-) create mode 100644 hypervisor/arch/arm-common/include/asm/coloring.h create mode 100644 hypervisor/arch/arm64/coloring.c create mode 100644 hypervisor/arch/arm64/include/asm/coloring.h create mode 100644 include/jailhouse/coloring.h diff --git a/hypervisor/arch/arm-common/include/asm/coloring.h b/hypervisor/arch/arm-common/include/asm/coloring.h new file mode 100644 index 00000000..91aa6285 --- /dev/null +++ b/hypervisor/arch/arm-common/include/asm/coloring.h @@ -0,0 +1,45 @@ +/* + * Jailhouse Cache Coloring Support - Stubs for ARMv7 + * + * Copyright (C) Technical University of Munich, 2020 + * + * Authors: + * Andrea Bastoni <andrea.bast...@tum.de> at https://rtsl.cps.mw.tum.de + * + * This work is licensed under the terms of the GNU GPL, version 2. See the + * COPYING file in the top-level directory. + */ +#ifndef _JAILHOUSE_ASM_COLORING_H +#define _JAILHOUSE_ASM_COLORING_H + +#include <jailhouse/utils.h> +#include <jailhouse/control.h> + +static inline void arm_color_dcache_flush_memory_region( + unsigned long phys, + unsigned long size, + unsigned long virt, + u64 color_mask, + enum dcache_flush flush_type) +{ + BUG(); +} + +static inline int +color_paging_create(const struct paging_structures *pg_structs, + unsigned long phys, unsigned long size, unsigned long virt, + unsigned long access_flags, unsigned long paging_flags, + u64 color_mask, u64 mem_flags) +{ + return -EINVAL; +} + +static inline int +color_paging_destroy(const struct paging_structures *pg_structs, + unsigned long phys, unsigned long size, unsigned long virt, + unsigned long paging_flags, u64 color_mask, u64 mem_flags) +{ + return -EINVAL; +} + +#endif diff --git a/hypervisor/arch/arm-common/include/asm/dcaches.h b/hypervisor/arch/arm-common/include/asm/dcaches.h index 87c316dc..26c5201a 100644 --- a/hypervisor/arch/arm-common/include/asm/dcaches.h +++ b/hypervisor/arch/arm-common/include/asm/dcaches.h @@ -11,6 +11,8 @@ * This work is licensed under the terms of the GNU GPL, version 2. See * the COPYING file in the top-level directory. */ +#ifndef _JAILHOUSE_DCACHES_H +#define _JAILHOUSE_DCACHES_H #ifndef __ASSEMBLY__ @@ -22,7 +24,13 @@ enum dcache_flush { DCACHE_CLEAN_AND_INVALIDATE, }; +void arm_dcache_flush_memory_region( + unsigned long region_addr, + unsigned long region_size, + enum dcache_flush flush); void arm_dcaches_flush(void *addr, unsigned long size, enum dcache_flush flush); void arm_cell_dcaches_flush(struct cell *cell, enum dcache_flush flush); #endif /* !__ASSEMBLY__ */ + +#endif diff --git a/hypervisor/arch/arm-common/mmu_cell.c b/hypervisor/arch/arm-common/mmu_cell.c index 7f9bb10a..4c7827ce 100644 --- a/hypervisor/arch/arm-common/mmu_cell.c +++ b/hypervisor/arch/arm-common/mmu_cell.c @@ -16,6 +16,7 @@ #include <asm/sysregs.h> #include <asm/control.h> #include <asm/iommu.h> +#include <asm/coloring.h> int arch_map_memory_region(struct cell *cell, const struct jailhouse_memory *mem) @@ -46,7 +47,12 @@ int arch_map_memory_region(struct cell *cell, if (err) return err; - err = paging_create(&cell->arch.mm, phys_start, mem->size, + if (mem->flags & JAILHOUSE_MEM_COLORED) + err = color_paging_create(&cell->arch.mm, phys_start, + mem->size, mem->virt_start, access_flags, + paging_flags, mem->colors, mem->flags); + else + err = paging_create(&cell->arch.mm, phys_start, mem->size, mem->virt_start, access_flags, paging_flags); if (err) iommu_unmap_memory_region(cell, mem); @@ -63,6 +69,11 @@ int arch_unmap_memory_region(struct cell *cell, if (err) return err; + if (mem->flags & JAILHOUSE_MEM_COLORED) + return color_paging_destroy(&cell->arch.mm, + mem->phys_start, mem->size, mem->virt_start, + PAGING_COHERENT, mem->colors, mem->flags); + return paging_destroy(&cell->arch.mm, mem->virt_start, mem->size, PAGING_COHERENT); } @@ -73,9 +84,33 @@ unsigned long arch_paging_gphys2phys(unsigned long gphys, unsigned long flags) return paging_virt2phys(&this_cell()->arch.mm, gphys, flags); } +void arm_dcache_flush_memory_region( + unsigned long region_addr, + unsigned long region_size, + enum dcache_flush flush) +{ + unsigned long size; + + while (region_size > 0) { + size = MIN(region_size, + NUM_TEMPORARY_PAGES * PAGE_SIZE); + + /* cannot fail, mapping area is preallocated */ + paging_create(&this_cpu_data()->pg_structs, region_addr, + size, TEMPORARY_MAPPING_BASE, + PAGE_DEFAULT_FLAGS, + PAGING_NON_COHERENT | PAGING_NO_HUGE); + + arm_dcaches_flush((void *)TEMPORARY_MAPPING_BASE, size, + flush); + + region_addr += size; + region_size -= size; + } +} + void arm_cell_dcaches_flush(struct cell *cell, enum dcache_flush flush) { - unsigned long region_addr, region_size, size; struct jailhouse_memory const *mem; unsigned int n; @@ -83,24 +118,17 @@ void arm_cell_dcaches_flush(struct cell *cell, enum dcache_flush flush) if (mem->flags & (JAILHOUSE_MEM_IO | JAILHOUSE_MEM_COMM_REGION)) continue; - region_addr = mem->phys_start; - region_size = mem->size; - - while (region_size > 0) { - size = MIN(region_size, - NUM_TEMPORARY_PAGES * PAGE_SIZE); - - /* cannot fail, mapping area is preallocated */ - paging_create(&this_cpu_data()->pg_structs, region_addr, - size, TEMPORARY_MAPPING_BASE, - PAGE_DEFAULT_FLAGS, - PAGING_NON_COHERENT | PAGING_NO_HUGE); - - arm_dcaches_flush((void *)TEMPORARY_MAPPING_BASE, size, - flush); - - region_addr += size; - region_size -= size; + if (mem->flags & JAILHOUSE_MEM_COLORED) { + arm_color_dcache_flush_memory_region( + mem->phys_start, + mem->size, + mem->virt_start, + mem->colors, + flush); + } else { + arm_dcache_flush_memory_region(mem->phys_start, + mem->size, + flush); } } diff --git a/hypervisor/arch/arm64/Kbuild b/hypervisor/arch/arm64/Kbuild index 4d328021..a5525811 100644 --- a/hypervisor/arch/arm64/Kbuild +++ b/hypervisor/arch/arm64/Kbuild @@ -23,3 +23,4 @@ lib-y := $(common-objs-y) lib-y += entry.o setup.o control.o mmio.o paging.o caches.o traps.o lib-y += iommu.o smmu-v3.o ti-pvu.o lib-y += smmu.o +lib-y += coloring.o diff --git a/hypervisor/arch/arm64/coloring.c b/hypervisor/arch/arm64/coloring.c new file mode 100644 index 00000000..5ec437f1 --- /dev/null +++ b/hypervisor/arch/arm64/coloring.c @@ -0,0 +1,108 @@ +/* + * Jailhouse Cache Coloring Support + * + * Copyright (C) Università di Modena e Reggio Emilia, 2018 + * Copyright (C) Boston University, 2020 + * Copyright (C) Technical University of Munich, 2020 + * + * Authors: + * Luca Miccio <lucmic...@gmail.com> + * Renato Mancuso (BU) <rmanc...@bu.edu> + * Andrea Bastoni <andrea.bast...@tum.de> at https://rtsl.cps.mw.tum.de + * + * This work is licensed under the terms of the GNU GPL, version 2. See the + * COPYING file in the top-level directory. + */ +#include <jailhouse/control.h> +#include <jailhouse/paging.h> +#include <jailhouse/printk.h> +#include <jailhouse/unit.h> +#include <jailhouse/cell.h> +#include <jailhouse/coloring.h> +#include <asm/coloring.h> + +/** + * Only parameter needed to determine the coloring. + */ +u64 coloring_way_size = 0; + +/** Temporary load-mapping parameter */ +u64 coloring_root_map_offset = 0; + +static int dispatch_op( + struct color_op *op, + unsigned long bphys, + unsigned long bvirt, + unsigned long bsize) +{ + if (op->op & (COL_OP_CREATE | COL_OP_LOAD)) { + if (op->op & COL_OP_LOAD) { + /* Fix addr to match the driver's IPA ioremap */ + bvirt += coloring_root_map_offset; + } + return paging_create(op->pg_structs, bphys, bsize, bvirt, + op->access_flags, op->paging_flags); + } + + if (op->op & (COL_OP_DESTROY | COL_OP_START)) { + if (op->op & COL_OP_START) { + /* Match the address specified during load */ + bvirt += coloring_root_map_offset; + } + return paging_destroy(op->pg_structs, bvirt, bsize, + op->paging_flags); + } + + if (op->op & COL_OP_FLUSH) { + arm_dcache_flush_memory_region(bphys, bsize, op->flush_type); + return 0; + } + + return -EINVAL; +} + +int color_do_op(struct color_op *op) +{ + unsigned long bvirt, bphys, bsize; + /* bit: start, low, contiguous bit range width */ + unsigned int bs, bl, bw; + unsigned int n; + u64 colors; + int err; + + col_print("[%c] OP 0x%x: P: 0x%08lx V: 0x%08lx " + "(S: 0x%lx C: 0x%08llx A: 0x%lx P: 0x%lx F: 0x%d)\n", + (op->pg_structs == &root_cell.arch.mm) ? 'r' : 'c', + op->op, op->phys, op->virt, op->size, op->color_mask, + op->access_flags, op->paging_flags, op->flush_type); + + n = 0; + bvirt = op->virt; + bphys = bsize = 0; + while (bvirt < op->virt + op->size) { + bs = bl = bw = 0; + colors = op->color_mask; + + while (colors != 0) { + /* update colors with next color-range */ + get_bit_range(&colors, &bl, &bw); + bs += bl; + bsize = bw * PAGE_SIZE; + bphys = op->phys + (bs * PAGE_SIZE) + + (n * coloring_way_size); + + err = dispatch_op(op, bphys, bvirt, bsize); + if (err) + return err; + + /* update next round */ + bvirt += bsize; + } + n++; + } + + col_print("end P: 0x%08lx V: 0x%08lx (bsize = 0x%08lx)\n", + bphys, bvirt - bsize, bsize); + + return err; +} diff --git a/hypervisor/arch/arm64/include/asm/coloring.h b/hypervisor/arch/arm64/include/asm/coloring.h new file mode 100644 index 00000000..233237fc --- /dev/null +++ b/hypervisor/arch/arm64/include/asm/coloring.h @@ -0,0 +1,155 @@ +/* + * Jailhouse Cache Coloring Support + * + * Copyright (C) Università di Modena e Reggio Emilia, 2018 + * Copyright (C) Boston University, 2020 + * Copyright (C) Technical University of Munich, 2020 + * + * Authors: + * Luca Miccio <lucmic...@gmail.com> + * Renato Mancuso (BU) <rmanc...@bu.edu> + * Andrea Bastoni <andrea.bast...@tum.de> at https://rtsl.cps.mw.tum.de + * + * This work is licensed under the terms of the GNU GPL, version 2. See the + * COPYING file in the top-level directory. + */ +#ifndef _JAILHOUSE_ASM_COLORING_H +#define _JAILHOUSE_ASM_COLORING_H + +#include <jailhouse/cell-config.h> +#include <jailhouse/utils.h> +#include <jailhouse/control.h> + +#ifdef CONFIG_DEBUG +#define col_print(fmt, ...) \ + printk("[COL] " fmt, ##__VA_ARGS__) +#else +#define col_print(fmt, ...) do { } while (0) +#endif + +/** Color operations */ +#define COL_OP_CREATE 0x1 +#define COL_OP_DESTROY 0x2 +#define COL_OP_START 0x4 +#define COL_OP_LOAD 0x8 +#define COL_OP_FLUSH 0x10 + +/** + * Only parameter needed to determine the coloring. + */ +extern u64 coloring_way_size; + +/** Temporary load-mapping parameter */ +extern u64 coloring_root_map_offset; + +/** + * Colored Operation + */ +struct color_op { + const struct paging_structures *pg_structs; + unsigned long phys; + unsigned long size; + unsigned long virt; + unsigned long access_flags; + unsigned long paging_flags; + u64 color_mask; + enum dcache_flush flush_type; + unsigned int op; +}; + +/** + * Colored operations on a cell / memory region. + * + * Encapsulate the loops needed to iterate through a region and identify + * the color-compatible phys2virt mappings. + */ +extern int color_do_op(struct color_op *op); + +static inline void arm_color_dcache_flush_memory_region( + unsigned long phys, + unsigned long size, + unsigned long virt, + u64 color_mask, + enum dcache_flush flush_type) +{ + struct color_op op; + + if (coloring_way_size == 0) + BUG(); + + op.phys = phys; + op.size = size; + op.virt = virt; + op.color_mask = color_mask; + op.flush_type = flush_type; + op.op = COL_OP_FLUSH; + + color_do_op(&op); +} + +/** + * Detection of coloring way size. + */ +static inline void arm_color_init(void) +{ + coloring_way_size = system_config->platform_info.color.way_size; + coloring_root_map_offset = + system_config->platform_info.color.root_map_offset; + + printk("Init Coloring: Way size: 0x%llx, TMP load addr: 0x%llx\n", + coloring_way_size, coloring_root_map_offset); +} + +static inline int +color_paging_create(const struct paging_structures *pg_structs, + unsigned long phys, unsigned long size, unsigned long virt, + unsigned long access_flags, unsigned long paging_flags, + u64 color_mask, u64 mem_flags) +{ + struct color_op op; + + if (coloring_way_size == 0) + return -EINVAL; + + op.pg_structs = pg_structs; + op.phys = phys; + op.size = size; + op.virt = virt; + op.access_flags = access_flags; + op.paging_flags = paging_flags; + op.color_mask = color_mask; + if (mem_flags & JAILHOUSE_MEM_TMP_ROOT_REMAP) { + op.op = COL_OP_LOAD; + } else { + op.op = COL_OP_CREATE; + } + + return color_do_op(&op); +} + +static inline int +color_paging_destroy(const struct paging_structures *pg_structs, + unsigned long phys, unsigned long size, unsigned long virt, + unsigned long paging_flags, u64 color_mask, u64 mem_flags) +{ + struct color_op op; + + if (coloring_way_size == 0) + return -EINVAL; + + op.pg_structs = pg_structs; + op.phys = phys; + op.size = size; + op.virt = virt; + op.paging_flags = paging_flags; + op.color_mask = color_mask; + if (mem_flags & JAILHOUSE_MEM_TMP_ROOT_REMAP) { + op.op = COL_OP_START; + } else { + op.op = COL_OP_DESTROY; + } + + return color_do_op(&op); +} + +#endif diff --git a/hypervisor/arch/arm64/setup.c b/hypervisor/arch/arm64/setup.c index a24bf1d7..548e49b7 100644 --- a/hypervisor/arch/arm64/setup.c +++ b/hypervisor/arch/arm64/setup.c @@ -20,6 +20,7 @@ #include <asm/irqchip.h> #include <asm/setup.h> #include <asm/smccc.h> +#include <asm/coloring.h> extern u8 __trampoline_start[]; @@ -40,6 +41,8 @@ int arch_init_early(void) if (err) return err; + arm_color_init(); + return arm_init_early(); } diff --git a/hypervisor/control.c b/hypervisor/control.c index 2214406f..d3b40c6a 100644 --- a/hypervisor/control.c +++ b/hypervisor/control.c @@ -300,7 +300,7 @@ static bool address_in_region(unsigned long addr, addr < (region->phys_start + region->size); } -static int unmap_from_root_cell(const struct jailhouse_memory *mem) +static int unmap_from_root_cell(const struct jailhouse_memory *mem, bool create) { /* * arch_unmap_memory_region and mmio_subpage_unregister use the @@ -316,6 +316,12 @@ static int unmap_from_root_cell(const struct jailhouse_memory *mem) return 0; } + if (!create && (mem->flags & JAILHOUSE_MEM_COLORED)) { + /* start cell: remove temporary mapping */ + tmp.flags |= JAILHOUSE_MEM_TMP_ROOT_REMAP; + tmp.virt_start = mem->virt_start; + } + return arch_unmap_memory_region(&root_cell, &tmp); } @@ -347,6 +353,17 @@ static int remap_to_root_cell(const struct jailhouse_memory *mem, overlap.phys_start - root_mem->phys_start; overlap.flags = root_mem->flags; + if (mem->flags & JAILHOUSE_MEM_COLORED) { + /* Use the colors from the to-be-remapped region */ + overlap.flags |= JAILHOUSE_MEM_COLORED; + overlap.colors = mem->colors; + if (mode == ABORT_ON_ERROR) { + /* load cell: setup temporary mapping */ + overlap.flags |= JAILHOUSE_MEM_TMP_ROOT_REMAP; + overlap.virt_start = mem->virt_start; + } + } + if (JAILHOUSE_MEMORY_IS_SUBPAGE(&overlap)) err = mmio_subpage_register(&root_cell, &overlap); else @@ -526,7 +543,7 @@ static int cell_create(struct per_cpu *cpu_data, unsigned long config_address) */ if (!(mem->flags & (JAILHOUSE_MEM_COMM_REGION | JAILHOUSE_MEM_ROOTSHARED))) { - err = unmap_from_root_cell(mem); + err = unmap_from_root_cell(mem, true); if (err) goto err_destroy_cell; } @@ -633,7 +650,7 @@ static int cell_start(struct per_cpu *cpu_data, unsigned long id) /* unmap all loadable memory regions from the root cell */ for_each_mem_region(mem, cell->config, n) if (mem->flags & JAILHOUSE_MEM_LOADABLE) { - err = unmap_from_root_cell(mem); + err = unmap_from_root_cell(mem, false); if (err) goto out_resume; } diff --git a/hypervisor/include/jailhouse/control.h b/hypervisor/include/jailhouse/control.h index f2b07c0d..03b0f3fd 100644 --- a/hypervisor/include/jailhouse/control.h +++ b/hypervisor/include/jailhouse/control.h @@ -9,6 +9,8 @@ * This work is licensed under the terms of the GNU GPL, version 2. See * the COPYING file in the top-level directory. */ +#ifndef _JAILHOUSE_CONTROL_H +#define _JAILHOUSE_CONTROL_H #include <jailhouse/bitops.h> #include <jailhouse/percpu.h> @@ -291,3 +293,5 @@ void __attribute__((noreturn)) arch_panic_stop(void); void arch_panic_park(void); /** @} */ + +#endif diff --git a/include/jailhouse/coloring.h b/include/jailhouse/coloring.h new file mode 100644 index 00000000..86158196 --- /dev/null +++ b/include/jailhouse/coloring.h @@ -0,0 +1,39 @@ +/* + * Jailhouse Cache Coloring Support + * + * Copyright (C) Technical University of Munich, 2020 + * + * Authors: + * Andrea Bastoni <andrea.bast...@tum.de> + * + * This work is licensed under the terms of the GNU GPL, version 2. See the + * COPYING file in the top-level directory. + */ +/** MSB/LSB function names differs between Jailhouse and Linux */ +#define _lsb(x) ffsl(x) +#define _msb(x) msbl(x) + +/** + * Get range of contiguous bits in a bitmask. + * + * The function returns: + * - bitmask without the extracted bit range. + * - low: original bit position of range start. + * - size: size of the range + * + * The function assumes bitmask is not 0. + */ +static inline void get_bit_range( + u64 *bitmask, + unsigned int *low, + unsigned int *size) +{ + unsigned int _range; + + *low = _lsb(*bitmask); + _range = *bitmask >> *low; + *bitmask = _range & (_range + 1UL); + + _range = _range ^ *bitmask; + *size = _msb(_range) + 1; +} -- 2.30.2 -- You received this message because you are subscribed to the Google Groups "Jailhouse" group. To unsubscribe from this group and stop receiving emails from it, send an email to jailhouse-dev+unsubscr...@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/jailhouse-dev/20210322194507.82643-5-andrea.bastoni%40tum.de.
