On Tue, 2022-07-12 at 12:08 -0700, Dan Williams wrote:
> Exercise the fundamental region provisioning sysfs mechanisms of discovering
> available DPA capacity, allocating DPA to a region, and programming HDM
> decoders.
>
> Signed-off-by: Dan Williams <[email protected]>
> ---
> test/cxl-region-create.sh | 122
> +++++++++++++++++++++++++++++++++++++++++++++
> test/meson.build | 2 +
> 2 files changed, 124 insertions(+)
> create mode 100644 test/cxl-region-create.sh
Since this isn't actually creating a region, should this be named
cxl-reserve-dpa.sh ?
Alternatively - I guess this test could just be extended to do actual
region creation once that is available in cxl-cli?
>
> diff --git a/test/cxl-region-create.sh b/test/cxl-region-create.sh
> new file mode 100644
> index 000000000000..389988759b08
> --- /dev/null
> +++ b/test/cxl-region-create.sh
> @@ -0,0 +1,122 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +# Copyright (C) 2022 Intel Corporation. All rights reserved.
> +
> +. $(dirname $0)/common
> +
> +rc=1
> +
> +set -ex
> +
> +trap 'err $LINENO' ERR
> +
> +check_prereq "jq"
> +
> +modprobe -r cxl_test
> +modprobe cxl_test
> +udevadm settle
> +
> +# THEORY OF OPERATION: Create a x8 interleave across the pmem capacity
> +# of the 8 endpoints defined by cxl_test, commit the decoders (which
> +# just stubs out the actual hardware programming aspect, but updates the
> +# driver state), and then tear it all down again. As with other cxl_test
> +# tests if the CXL topology in tools/testing/cxl/test/cxl.c ever changes
> +# then the paired update must be made to this test.
> +
> +# find the root decoder that spans both test host-bridges and support pmem
> +decoder=$(cxl list -b cxl_test -D -d root | jq -r ".[] |
This and all following 'cxl' invocations should be "$CXL" which we get
from test/common
> + select(.pmem_capable == true) |
> + select(.nr_targets == 2) |
> + .decoder")
> +
> +# find the memdevs mapped by that decoder
> +readarray -t mem < <(cxl list -M -d $decoder | jq -r ".[].memdev")
> +
> +# ask cxl reserve-dpa to allocate pmem capacity from each of those memdevs
> +readarray -t endpoint < <(cxl reserve-dpa -t pmem ${mem[*]} -s $((256<<20)) |
> + jq -r ".[] | .decoder.decoder")
> +
> +# instantiate an empty region
> +region=$(cat /sys/bus/cxl/devices/$decoder/create_pmem_region)
> +echo $region > /sys/bus/cxl/devices/$decoder/create_pmem_region
> +uuidgen > /sys/bus/cxl/devices/$region/uuid
> +
> +# setup interleave geometry
> +nr_targets=${#endpoint[@]}
> +echo $nr_targets > /sys/bus/cxl/devices/$region/interleave_ways
> +g=$(cat /sys/bus/cxl/devices/$decoder/interleave_granularity)
> +echo $g > /sys/bus/cxl/devices/$region/interleave_granularity
> +echo $((nr_targets * (256<<20))) > /sys/bus/cxl/devices/$region/size
> +
> +# grab the list of memdevs grouped by host-bridge interleave position
> +port_dev0=$(cxl list -T -d $decoder | jq -r ".[] |
> + .targets | .[] | select(.position == 0) | .target")
> +port_dev1=$(cxl list -T -d $decoder | jq -r ".[] |
> + .targets | .[] | select(.position == 1) | .target")
> +readarray -t mem_sort0 < <(cxl list -M -p $port_dev0 | jq -r ".[] | .memdev")
> +readarray -t mem_sort1 < <(cxl list -M -p $port_dev1 | jq -r ".[] | .memdev")
> +
> +# TODO: add a cxl list option to list memdevs in valid region provisioning
> +# order, hardcode for now.
> +mem_sort=()
> +mem_sort[0]=${mem_sort0[0]}
> +mem_sort[1]=${mem_sort1[0]}
> +mem_sort[2]=${mem_sort0[2]}
> +mem_sort[3]=${mem_sort1[2]}
> +mem_sort[4]=${mem_sort0[1]}
> +mem_sort[5]=${mem_sort1[1]}
> +mem_sort[6]=${mem_sort0[3]}
> +mem_sort[7]=${mem_sort1[3]}
> +
> +# TODO: use this alternative memdev ordering to validate a negative test for
> +# specifying invalid positions of memdevs
> +#mem_sort[2]=${mem_sort0[0]}
> +#mem_sort[1]=${mem_sort1[0]}
> +#mem_sort[0]=${mem_sort0[2]}
> +#mem_sort[3]=${mem_sort1[2]}
> +#mem_sort[4]=${mem_sort0[1]}
> +#mem_sort[5]=${mem_sort1[1]}
> +#mem_sort[6]=${mem_sort0[3]}
> +#mem_sort[7]=${mem_sort1[3]}
> +
> +# re-generate the list of endpoint decoders in region position programming
> order
> +endpoint=()
> +for i in ${mem_sort[@]}
> +do
> + readarray -O ${#endpoint[@]} -t endpoint < <(cxl list -Di -d endpoint
> -m $i | jq -r ".[] |
> + select(.mode ==
> \"pmem\") | .decoder")
> +done
> +
> +# attach all endpoint decoders to the region
> +pos=0
> +for i in ${endpoint[@]}
> +do
> + echo $i > /sys/bus/cxl/devices/$region/target$pos
> + pos=$((pos+1))
> +done
> +echo "$region added ${#endpoint[@]} targets: ${endpoint[@]}"
> +
> +# walk up the topology and commit all decoders
> +echo 1 > /sys/bus/cxl/devices/$region/commit
> +
> +# walk down the topology and de-commit all decoders
> +echo 0 > /sys/bus/cxl/devices/$region/commit
> +
> +# remove endpoints from the region
> +pos=0
> +for i in ${endpoint[@]}
> +do
> + echo "" > /sys/bus/cxl/devices/$region/target$pos
> + pos=$((pos+1))
> +done
> +
> +# release DPA capacity
> +readarray -t endpoint < <(cxl free-dpa -t pmem ${mem[*]} |
> + jq -r ".[] | .decoder.decoder")
> +echo "$region released ${#endpoint[@]} targets: ${endpoint[@]}"
> +
> +# validate no WARN or lockdep report during the run
> +log=$(journalctl -r -k --since "-$((SECONDS+1))s")
> +grep -q "Call Trace" <<< $log && err "$LINENO"
> +
> +modprobe -r cxl_test
> diff --git a/test/meson.build b/test/meson.build
> index 210dcb0b5ff1..fbcfc08d03ee 100644
> --- a/test/meson.build
> +++ b/test/meson.build
> @@ -151,6 +151,7 @@ max_extent = find_program('max_available_extent_ns.sh')
> pfn_meta_errors = find_program('pfn-meta-errors.sh')
> track_uuid = find_program('track-uuid.sh')
> cxl_topo = find_program('cxl-topology.sh')
> +cxl_region = find_program('cxl-region-create.sh')
>
> tests = [
> [ 'libndctl', libndctl, 'ndctl' ],
> @@ -176,6 +177,7 @@ tests = [
> [ 'pfn-meta-errors.sh', pfn_meta_errors, 'ndctl' ],
> [ 'track-uuid.sh', track_uuid, 'ndctl' ],
> [ 'cxl-topology.sh', cxl_topo, 'cxl' ],
> + [ 'cxl-region-create.sh', cxl_region, 'cxl' ],
> ]
>
> if get_option('destructive').enabled()
>
