omjavaid updated this revision to Diff 286161.
omjavaid added a comment.
Herald added a subscriber: JDevlieghere.
In light of review comments I have come up with this new sceheme for setting up
QEMU VM based testing environment for LLDB.
We now have three scripts:
1. setup.sh is used to build Linux kernel images for Arm and AArch64. Also it
builds QEMU from latest sources.
2. rootfs.sh can generate Ubuntu rootf file system images for our QEMU VMs
3. run-qemu.sh puts it all together and uses kernel image, rootfs image and
arch information to run a QEMU VM. We currently support Arm and AArch64
architectures and more architectures can be added later.
This can generates a QEMU testing setup in less than 10 minutes on a host
machine with reasonable resources.
@labath @rovka thoughts?
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D82064/new/
https://reviews.llvm.org/D82064
Files:
lldb/tools/lldb-test-qemu/README.txt
lldb/tools/lldb-test-qemu/rootfs.sh
lldb/tools/lldb-test-qemu/run-qemu.sh
lldb/tools/lldb-test-qemu/setup.sh
Index: lldb/tools/lldb-test-qemu/setup.sh
===================================================================
--- /dev/null
+++ lldb/tools/lldb-test-qemu/setup.sh
@@ -0,0 +1,151 @@
+#!/bin/bash
+
+print_usage() {
+ echo "Usage: $(basename $0) [options]"
+ echo -e "Builds QEMU and Linux kernel from source.\n"
+ echo -e " --help\t\t\tDisplay this information."
+ echo -e " --kernel {arm|arm64}\t\tBuild Linux kernel for the architecture."
+ echo -e " --qemu\t\t\tBuild QEMU from source."
+ echo -e " --clean\t\t\tRemove qemu.git and linux.git directories in current directory."
+ exit "$1"
+}
+
+update_repositories() {
+ echo -e "\nUpdating apt repositories. "
+ echo -e "\nPress 'y' to continue or any other key to exit..."
+ read -s -n 1 user_input
+ if [[ $user_input == 'Y' ]] || [[ $user_input == 'y' ]]; then
+ sudo apt update
+ else
+ exit
+ fi
+}
+
+check_dir_exists() {
+ user_input=
+ if [ -d "$1" ]; then
+ echo -e "\n$1 already exists in working directory and will not be updated."
+ echo -e "\nPress 'y' to continue or any other key to exit..."
+ read -s -n 1 user_input
+ if [[ $user_input != 'Y' ]] && [[ $user_input != 'y' ]]; then
+ exit
+ fi
+ fi
+}
+
+invalid_arg() {
+ echo "ERROR: Unrecognized argument: $1" >&2
+ print_usage 1
+}
+
+build_qemu() {
+ echo "Installing QEMU build dependencies ..."
+ sudo apt install git python3-dev libsdl1.2-dev build-essential libpixman-1-dev
+
+ # Checkout source code
+ check_dir_exists "qemu.git"
+ if [ ! -d "qemu.git" ]; then
+ git clone --depth 1 git://git.qemu.org/qemu.git qemu.git
+ fi
+
+ cd qemu.git
+ # We are going to build QEMU Arm and AArch64 system mode emulation.
+ # ./configure --help emits a list of other possible targets supported by QEMU.
+ ./configure --target-list=arm-softmmu,aarch64-softmmu
+ make -j`getconf _NPROCESSORS_ONLN`
+}
+
+build_linux() {
+ echo "Installing Linux kernel build dependencies ..."
+ sudo apt install git bison flex build-essential libssl-dev bc
+
+ check_dir_exists "linux.git"
+
+ if [ ! -d "linux.git" ]; then
+ git clone --depth 1 \
+ https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git linux.git
+ fi
+
+ cd linux.git
+ make mrproper
+
+ if [[ "$1" == "arm" ]]; then
+ echo "Installing gcc-arm-linux-gnueabihf ..."
+ sudo apt install gcc-arm-linux-gnueabihf
+
+ # Configure kernel_branch=master arch=arm config=vexpress_defconfig
+ make O=../linux.build/arm ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \
+ vexpress_defconfig
+
+ # Trigger Arm kernel build
+ make -j`getconf _NPROCESSORS_ONLN` O=../linux.build/arm ARCH=arm \
+ CROSS_COMPILE=arm-linux-gnueabihf-
+ elif [[ "$1" == "arm64" ]]; then
+ echo "Installing gcc-aarch64-linux-gnu ..."
+ sudo apt install gcc-aarch64-linux-gnu
+
+ # Configure kernel_branch=master arch=arm64 config=defconfig
+ make O=../linux.build/arm64 ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- \
+ defconfig
+
+ # Trigger AArch64 kernel build
+ make -j`getconf _NPROCESSORS_ONLN` O=../linux.build/arm64 ARCH=arm64 \
+ CROSS_COMPILE=aarch64-linux-gnu-
+ else
+ echo "ERROR: Unrecognized architecture: $1" >&2
+ print_usage 1
+ exit
+ fi
+}
+
+clean() {
+ if [ -d "linux.git" ]; then
+ echo "Removing linux.git ..."
+ rm -rf linux.git
+ fi
+
+ if [ -d "linux.build" ]; then
+ echo "Removing linux.build ..."
+ rm -rf linux.build
+ fi
+
+ if [ -d "qemu.git" ]; then
+ echo "Removing qemu.git ..."
+ rm -rf qemu.git
+ fi
+
+ exit
+}
+
+# Parse options
+while [[ $# -gt 0 ]]; do
+ case "${END_OF_OPT}${1}" in
+ -h|--help) print_usage 0 ;;
+ -k|--kernel)
+ if [ "$2" == "arm64" ] || [ "$2" == "arm" ]; then
+ KERNEL_ARCH=$2
+ else
+ invalid_arg "$2"
+ fi
+ shift;;
+ -q|--qemu)
+ QEMU=1;;
+ -c|--clean) clean ;;
+ *) invalid_arg "$1" ;;
+ esac
+ shift
+done
+
+update_repositories
+
+if [ "$KERNEL_ARCH" != "" ]; then
+ pushd .
+ build_linux $KERNEL_ARCH
+ popd
+fi
+
+if [[ $QEMU -eq 1 ]]; then
+ pushd .
+ build_qemu
+ popd
+fi
Index: lldb/tools/lldb-test-qemu/run-qemu.sh
===================================================================
--- /dev/null
+++ lldb/tools/lldb-test-qemu/run-qemu.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+print_usage() {
+ echo "Usage: $(basename $0) --arch [arm|arm64] [options]"
+ echo -e "Starts QEMU system mode emulation for the architecture.\n"
+ echo -e " --help\t\t\tDisplay this information."
+ echo -e " --arch {arm|arm64}\t\tSelects architecture QEMU system emulation."
+ echo -e " --sve {path}\t\t\tEnables AArch64 SVE mode.\n"
+ echo -e " --rootfs {path}\t\tPath of root file system image."
+ echo -e " --qemu {path}\t\t\tPath of pre-installed qemu-system-* executable."
+ echo -e " --kernel {path}\t\tPath of Linux kernel prebuilt image.\n"
+ echo -e "By default this utility will use:"
+ echo -e " QEMU image built from source in qemu.git directory"
+ echo -e " Linux kernel image from linux.build/(arm or arm64) directory."
+ echo -e "Custom Linux kernel image or QEMU binary can be provided using commandline."
+ exit "$1"
+}
+
+invalid_arg() {
+ echo "ERROR: Unrecognized argument: $1" >&2
+ print_usage 1
+}
+
+run_qemu() {
+ QEMU_CORES=2
+ QEMU_MEMORY=1024
+
+ $QEMU_BIN \
+ -cpu $QEMU_CPU \
+ -m $QEMU_MEMORY \
+ -smp $QEMU_CORES \
+ -kernel $KERNEL_IMG \
+ -machine $QEMU_MACHINE \
+ -drive file=$ROOTFS_IMG,if=none,format=raw,id=hd0 \
+ -device virtio-blk-device,drive=hd0 \
+ -append "root=/dev/vda rw ip=dhcp mem=1024M raid=noautodetect \
+ crashkernel=128M rootwait console=ttyAMA0 devtmpfs.mount=0" \
+ -netdev type=tap,id=net0 \
+ -device virtio-net-device,netdev=net0 \
+ -nographic
+}
+
+# Parse options
+while [[ $# -gt 0 ]]; do
+ case "${END_OF_OPT}${1}" in
+ --arch) ARCH=$2; shift;;
+ --rootfs) ROOTFS_IMG=$2; shift;;
+ --kernel) KERNEL_IMG=$2; shift;;
+ --qemu) QEMU_BIN=$2; shift;;
+ --sve) SVE=1;;
+ --help) print_usage 0 ;;
+ *) invalid_arg "$1" ;;
+ esac
+ shift
+done
+
+if [ "$ARCH" == "arm64" ] && [ "$ARCH" == "arm" ]; then
+ echo "Invalid architecture: $ARCH"
+ print_usage 1
+fi
+
+if [[ ! -f "$ROOTFS_IMG" ]]; then
+ echo "No root file system image image available for emulation."
+ exit
+fi
+
+if [[ ! -f "$KERNEL_IMG" ]]; then
+ KERNEL_IMG_PATH=$(pwd)/linux.build/"$ARCH"/arch/"$ARCH"/boot/
+
+ if [[ ! -d "$KERNEL_IMG_PATH" ]]; then
+ echo "No Linux kernel image available for emulation."
+ exit
+ fi
+
+ if [[ "$ARCH" == "arm" ]]; then
+ KERNEL_IMG=$KERNEL_IMG_PATH/zImage
+ elif [[ "$ARCH" == "arm64" ]]; then
+ KERNEL_IMG=$KERNEL_IMG_PATH/Image
+ fi
+fi
+
+if [[ ! -f "$QEMU_BIN" ]]; then
+ if [[ "$ARCH" == "arm" ]]; then
+ QEMU_BIN=$(pwd)/qemu.git/arm-softmmu/qemu-system-arm
+ elif [[ "$ARCH" == "arm64" ]]; then
+ QEMU_BIN=$(pwd)/qemu.git/aarch64-softmmu/qemu-system-aarch64
+ fi
+
+ if [[ ! -f "$QEMU_BIN" ]]; then
+ echo "QEMU $ARCH system emulation executable not found."
+ exit
+ fi
+fi
+
+if [[ "$ARCH" == "arm" ]]; then
+ QEMU_MACHINE="virt,highmem=off"
+ QEMU_CPU="cortex-a15"
+
+ if [[ $SVE ]]; then
+ invalid_arg "--sve"
+ fi
+elif [[ "$ARCH" == "arm64" ]]; then
+ QEMU_MACHINE=virt
+ QEMU_SVE_MAX_VQ=4
+ QEMU_CPU="cortex-a53"
+
+ if [[ $SVE ]]; then
+ QEMU_CPU="max,sve-max-vq=$QEMU_SVE_MAX_VQ"
+ fi
+fi
+
+run_qemu
Index: lldb/tools/lldb-test-qemu/rootfs.sh
===================================================================
--- /dev/null
+++ lldb/tools/lldb-test-qemu/rootfs.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+set -e
+
+print_usage() {
+ echo "Usage:"
+ echo "Usage: $(basename $0) [options]"
+ echo -e "Creates a Ubuntu root file system image.\n"
+ echo -e " --help\t\t\tDisplay this information."
+ echo -e " --arch {armhf|arm64}\t\tSelects architecture of rootfs image."
+ echo -e " --distro {bionic|focal}\tSelects Ubuntu distribution of rootfs image."
+ echo -e " --size n{K|M|G}\t\tSets size of rootfs image to n Kilo, Mega or Giga bytes."
+ exit "$1"
+}
+
+invalid_arg() {
+ echo "ERROR: Unrecognized argument: $1" >&2
+ print_usage 1
+}
+
+update_repositories() {
+ echo -e "\nUpdating apt repositories. "
+ echo -e "\nPress 'y' to continue or any other key to exit..."
+ read -s -n 1 user_input
+ if [[ $user_input == 'Y' ]] || [[ $user_input == 'y' ]]; then
+ sudo apt update
+ else
+ exit
+ fi
+}
+
+# Parse options
+while [[ $# -gt 0 ]]; do
+ case "${END_OF_OPT}${1}" in
+ --help) print_usage 0 ;;
+ --arch) rfs_arch=$2; shift;;
+ --distro) rfs_distro=$2; shift;;
+ --size) rfs_size=$2; shift;;
+ *) invalid_arg "$1" ;;
+ esac
+ shift
+done
+
+if [ -z "$rfs_arch" ]; then
+ echo "Missing architecture"
+ print_usage 1
+fi
+if [ -z "$rfs_distro" ]; then
+ echo "Missing distribution"
+ print_usage 1
+fi
+if [ -z "$rfs_size" ]; then
+ echo "Missing size"
+ print_usage 1
+fi
+
+if [[ "$rfs_arch" != "arm64" && "$rfs_arch" != "armhf" ]]; then
+ echo "Invalid architecture: $rfs_arch"
+ print_usage 1
+fi
+
+if [[ "$rfs_distro" != "focal" && "$rfs_distro" != "bionic" ]]; then
+ echo "Invalid distribution: $rfs_distro"
+ print_usage 1
+fi
+
+pat='^[0-9]+[K|M|G]$'
+if [[ ! $rfs_size =~ $pat ]]; then
+ echo "Invalid size: $rfs_size"
+ print_usage 1
+fi
+
+update_repositories
+
+echo "Installing build dependencies ..."
+sudo apt-get install debootstrap qemu-user-static schroot qemu-utils
+
+image_name=$rfs_distro-$rfs_arch-"rootfs"
+echo "Creating $rfs_distro ($rfs_arch) root file system ..."
+echo "Image name: $image_name.img"
+echo "Image size: $rfs_size"
+
+qemu-img create $image_name.img $rfs_size
+
+mkfs.ext4 $image_name.img
+mkdir $image_name.dir
+sudo mount -o loop $image_name.img $image_name.dir
+
+sudo qemu-debootstrap --arch $rfs_arch $rfs_distro $image_name.dir
+
+sudo chroot $image_name.dir locale-gen en_US.UTF-8
+
+sudo chroot $image_name.dir sed -i \
+'s/main/main restricted multiverse universe/g' /etc/apt/sources.list
+
+sudo chroot $image_name.dir sed -i '$ a\nameserver 8.8.8.8' /etc/resolv.conf
+
+sudo chroot $image_name.dir apt update
+sudo chroot $image_name.dir apt -y install ssh bash-completion
+sudo chroot $image_name.dir adduser --gecos "" $USER
+sudo chroot $image_name.dir adduser $USER sudo
+sudo umount $image_name.dir
+rmdir $image_name.dir
Index: lldb/tools/lldb-test-qemu/README.txt
===================================================================
--- /dev/null
+++ lldb/tools/lldb-test-qemu/README.txt
@@ -0,0 +1,119 @@
+----------------------------------------------------------------------
+ LLDB testing on Arm/AArch64 Linux using QEMU system mode emulation
+----------------------------------------------------------------------
+
+QEMU can be used to test LLDB in an emulation environment in the absence of
+actual hardware. This documents will help setup a QEMU system mode emulation
+environment for testing LLDB.
+
+Main motivation for writing this guide is to test AArch64 features like SVE,
+MTE, Pointer Authentication etc. These scripts can be used as reference to set
+up similar testing environment for other architectures supported by QEMU.
+
+We have written helper scripts under llvm-project/lldb/tools/lldb-test-qemu
+which can help quickly setup a Arm or AArch64 testing environment using QEMU.
+
+* setup.sh is used to build AArch64 and Arm Linux kernel image and QEMU
+ system emulation executable(s) from source.
+
+* rootfs.sh is used to generate Ubuntu root file system images to be used for
+ QEMU Arm and AArch64 system emulation.
+
+* run-qemu.sh utilizes QEMU to boot an Arm or AArch64 Linux kernel image with
+ a given root file system image.
+
+Once we have booted our kernel we can run lldb-server in emulation environment.
+Ubuntu Bionic/Focal x86_64 host was used to test all the instructions in this
+document. Please update it according to your host distribution/architecture.
+
+Note: sudo permissions are needed to run above scripts.
+
+Given below are some common examples of common use-cases of LLDB QEMU testing
+helper scripts:
+
+--------------------------------------------------------------------------------
+ Create Ubuntu root file system image for QEMU system emulation with rootfs.sh
+--------------------------------------------------------------------------------
+
+* Example: generate Ubuntu Bionic (armhf) rootfs image of size 1 GB
+ bash rootfs.sh --arch armhf --distro bionic --size 1G
+
+* Example: generate Ubuntu Focal (arm64) rootfs image of size 2 GB
+ bash rootfs.sh --arch arm64 --distro focal --size 2G
+
+* rootfs.sh currently support Ubuntu Bionic and Focal images but support for
+ various types of rootfs images can be added later.
+
+* rootfs.sh defaults username of generated image to your current username on
+ host computer.
+
+-----------------------------------------------------------------------
+ Build QEMU or cross compile Linux kernel from source using setup.sh
+-----------------------------------------------------------------------
+
+* Example: Build QEMU binaries and Arm/AArch64 Linux kernel image
+ bash setup.sh --qemu --kernel arm
+ bash setup.sh --qemu --kernel arm64
+
+* Example: Build Linux kernel image only
+ bash setup.sh --kernel arm
+ bash setup.sh --kernel arm64
+
+* Example: Build qemu-system-arm and qemu-system-aarch64 binaries.
+ bash setup.sh --qemu
+
+* Example: Remove qemu.git, linux.git and linux.build from working directory
+ bash setup.sh --clean
+
+--------------------------------------------------------------
+ Run QEMU Arm or AArch64 system emulation using run-qemu.sh
+--------------------------------------------------------------
+run-qemu.sh has following dependencies:
+
+* Follow https://wiki.qemu.org/Documentation/Networking/NAT and set up bridge
+ networking for QEMU.
+
+* Make sure /etc/qemu-ifup script is available with executable permissions.
+
+* QEMU binaries must be built from source using setup.sh or provided via --qemu
+ commandline argument.
+
+* Arm or AArch64 Linux kernel image must be built from source using setup.sh or
+ provided via --kernel commandline argument.
+
+* linux.build and qemu.git folder must be present in current directory if
+ setup.sh was used to build Linux kernel and QEMU binaries.
+
+* --sve option will enable AArch64 SVE mode.
+
+* Example: Run QEMU Arm or AArch64 system emulation using run-qemu.sh
+ sudo bash run-qemu.sh --arch arm --rootfs <path of rootfs image>
+ sudo bash run-qemu.sh --arch arm64 --rootfs <path of rootfs image>
+
+* Example: Run QEMU with kernel image and qemu binary provided using commandline
+ sudo bash run-qemu.sh --arch arm64 --rootfs <path of rootfs image> \
+ --kernel <path of Linux kernel image> --qemu <path of QEMU binary>
+
+----------------------------------------------------------------------
+ Steps for running lldb-server in QEMU system emulation environment
+----------------------------------------------------------------------
+
+1) Make sure bridge networking is enabled between host machine and QEMU VM
+
+2) Find out ip address assigned to eth0 in emulation environment
+
+3) Setup ssh access between host machine and emulation environment
+
+4) Login emulation environment and install dependencies
+ sudo apt install python-dev libedit-dev libncurses5-dev libexpat1-dev
+
+5) Cross compile LLDB server for AArch64 Linux
+ Please visit https://lldb.llvm.org/resources/build.html for instructions
+ on how to cross compile LLDB server.
+
+6) Transfer LLDB server executable to emulation environment
+ scp lldb-server username@ip-address-of-emulation-environment:/home/username
+
+7) Run lldb-server inside QEMU VM
+
+8) Try connecting to lldb-server running inside QEMU VM with selected ip:port
_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
