From: Prarit Bhargava <pra...@redhat.com> kabi: add stablelist helpers
Add helper scripts to update the kabi stablelist checksums and to diff symtypes output. redhat/kabi/diff-kabi computes the current symtypes files and compares it to the reference symtypes stored in the stablelist. redhat/kabi/update-kabi computes the current checksums and reference symtypes files and updates the stablelist contents. It does not stage any changes. redhat/kabi/symtypes computes symtypes image, preimage or a diff for a symtypes node. Signed-off-by: Čestmír Kalina <ckal...@redhat.com> Signed-off-by: Prarit Bhargava <pra...@redhat.com> diff --git a/redhat/kabi/diff-kabi b/redhat/kabi/diff-kabi new file mode 100755 index blahblah..blahblah 100755 --- /dev/null +++ b/redhat/kabi/diff-kabi @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +usage_desc() { + sed 's/^\t//' <<EOF + Diffs the kernel ABI stablelist symtypes information and current symtypes + for architecture ARCH. +EOF +} + +# Called whenever a new symbol checksum is obtained +# cb_checksum ARCH SYMBOL CHECKSUM FILE SYMVERSIONS_FILE SYMTYPES_FILE +cb_checksum() { + local arch="$1" + local symbol="$2" + local file="$4" + local symtypes="$6" + + if [ ! -e $REDHAT/kabi/kabi-module/kabi_$arch/.$symbol ]; then + echo "Reference file for $symbol on $arch not found." + return + fi + + $REDHAT/kabi/symtypes diff -s $symbol \ + $REDHAT/kabi/kabi-module/kabi_$arch/.$symbol $symtypes + + symref=${file:0:-1}symref + rm -f $symref || : + ln -s $(realpath $REDHAT/kabi/kabi-module/kabi_$arch/.$symbol) $symref + make -s ${MAKE_ARGS[@]} $symtypes 2>&1 | grep "warning: $symbol:" || : + rm -f $symref || : +} + +# Called whenever the generate command finished successfully +cb_ready() { + : +} + +cd "$(git rev-parse --show-toplevel)" +REDHAT=${REDHAT:-$(pwd)/redhat/} + +# Generate new symbol checksums and symvers files +. $REDHAT/kabi/symtype-generate diff --git a/redhat/kabi/symtype-generate b/redhat/kabi/symtype-generate new file mode 100755 index blahblah..blahblah 100755 --- /dev/null +++ b/redhat/kabi/symtype-generate @@ -0,0 +1,678 @@ +#!/usr/bin/env bash + +unset MAKEFLAGS +unset CPP +unset ARCH +unset SYMBOL + +set -euo pipefail + +usage() { + sed 's/^\t//' <<EOF + Usage: $0 [-h] [-v] [-1] [-2] [-3] [-4] [-5] -a ARCH [-s SYMBOL]... [FILE]... +EOF + + echo + [ "$(type -t usage_desc || :)" == "function" ] && usage_desc + echo + + sed 's/^\t//' <<EOF + -a ARCH Supported architectures: + x86_64, s390x, aarch64, ppc64le + + -s SYMBOL [-s SYMBOL ...] Symbol entry to update. + Updates the whole stablelist if omitted. + + -h Prints this message. + + -v Verbose output. + + == DESCRIPTION + + To calculate current symbol data, the file exporting the symbol on a given + arch must first be located. The following methods are attempted in order: + + 1. user-supplied FILEs, if given, (disable using -1) + 2. stablelist-supplied FILEs, (disable using -2) + 3. cscope, if available, (disable using -3) + 4. naive regular expression search for exporting macro (disable + using -4), + 5. compute all symvers and symtypes files (disable using -5) + 6. compute all symvers and symtypes files following a local kernel + build (disable using -6). + + If a method is successful in updating all of the required symbols, + the script early terminates. Methods are sorted by their time complexity + as well as generality. + + If you're using the tool directly, please make sure to commit/stash your + changes. + + You must also make sure that you have cross compilation toolchain + installed when computing non-native checksums. The tool uses + CROSS_COMPILE=/usr/bin/ARCH-linux-gnu-, override + KABI_CROSS_COMPILE_PREFIX=/usr/bin/ + KABI_CROSS_COMPILE_SUFFIX=-linux-gnu- + as needed. + + == NOTES + + Method 3 can only find symbols exported using one of the following + macros: + + ACPI_EXPORT_SYMBOL + ACPI_EXPORT_SYMBOL_INIT + EXPORT_DATA_SYMBOL + EXPORT_DATA_SYMBOL_GPL + EXPORT_INDIRECT_CALLABLE + EXPORT_PER_CPU_SYMBOL + EXPORT_PER_CPU_SYMBOL_GPL + EXPORT_STATIC_CALL + EXPORT_STATIC_CALL_GPL + EXPORT_STATIC_CALL_TRAMP + EXPORT_STATIC_CALL_TRAMP_GPL + EXPORT_SYMBOL + EXPORT_SYMBOL_FOR_TESTS_ONLY + EXPORT_SYMBOL_GPL + EXPORT_SYMBOL_NS + EXPORT_SYMBOL_NS_GPL + EXPORT_TRACEPOINT_SYMBOL + EXPORT_TRACEPOINT_SYMBOL_GPL + + and the symbol name must not be constructed using preprocessor + concatenation. +EOF + exit +} + +# --- decls + +# list of temporary files created during execution +declare -ag TEMP_FILES + +# list of makefile targets to invoke +declare -ag TARGETS + +# list of source files to process +declare -ag SOURCES + +# list of symbols to try and generate symbol versions/symtypes for +declare -ag SYMBOL + +# list of symbols that couldn't have been resolved using previous method +declare -ag SYMFAIL + +# current arch +declare -g CARCH + +# dict[arch]=ARCH variable for kernel makefile +declare -Ag CC_ARCH + +# dict[symbol]=checksum +# dict[symbol]=file_exporting_symbol +declare -Ag SYMCRC +declare -Ag SYMFILE +declare -Ag SYMFILE_T +declare -Ag SYMFILE_V + +# kernel makefile cc var +declare -g CROSS_COMPILE + +# compiler +declare -g CPP + +# verbose level +declare -g V + +# --- default assignments + +V=${V:-0} +CPP=gcc +KABI_CROSS_COMPILE_PREFIX="${KABI_CROSS_COMPILE_PREFIX:-/usr/bin/}" +KABI_CROSS_COMPILE_SUFFIX="${KABI_CROSS_COMPILE_SUFFIX:--linux-gnu-}" +USE_ENTIRE_STABLELIST=1 +MAKE_ARGS=(-k -i -j$(nproc)) +CC_ARCH[s390x]="s390" +CC_ARCH[x86_64]="x86" +CC_ARCH[ppc64le]="powerpc" +CC_ARCH[aarch64]="arm64" + +# --- helper fns + +echo0() { echo " :: $*"; } +echo1() { echo " $*"; } +echo2() { echo " - $*"; } +echo3() { echo " $*"; } +err() { echo -e "ERROR:" "$@" >&2; } +warn() { echo -e "WARNING:" "$@" >&2; } + +cleanup() { + [ -z "${TEMP_FILES[@]:-}" ] && return + echo0 "Cleaning up temporary files ..." + rm -f "${TEMP_FILES[@]}" +} + +# find files containing symbol export statements of the form MACRO(NAME), +# where symbol name NAME is the entire symbol name and MACRO is one of: +# ACPI_EXPORT_SYMBOL +# ACPI_EXPORT_SYMBOL_INIT +# EXPORT_DATA_SYMBOL +# EXPORT_DATA_SYMBOL_GPL +# EXPORT_INDIRECT_CALLABLE +# EXPORT_PER_CPU_SYMBOL +# EXPORT_PER_CPU_SYMBOL_GPL +# EXPORT_STATIC_CALL +# EXPORT_STATIC_CALL_GPL +# EXPORT_STATIC_CALL_TRAMP +# EXPORT_STATIC_CALL_TRAMP_GPL +# EXPORT_SYMBOL +# EXPORT_SYMBOL_FOR_TESTS_ONLY +# EXPORT_SYMBOL_GPL +# EXPORT_SYMBOL_NS +# EXPORT_SYMBOL_NS_GPL +# EXPORT_TRACEPOINT_SYMBOL +# EXPORT_TRACEPOINT_SYMBOL_GPL +# export_try_find SYMBOL|REGEX +export_tryfind() { + grep -Prol --exclude-dir=redhat "^(ACPI_)?EXPORT_(DATA_|INDIRECT_CALLABLE|STATIC_CALL|TRACEPOINT_)?(PER_CPU_)?(SYMBOL|_TRAMP)?(_NS)?(_GPL|_INIT)?\($1(,[^)]*)?\);" || : +} + +symbol_tryfind_symversions() { + # __crc_SYMBOL = CHECKSUM + # SECTIONS { .rodata : ALIGN(4) { __crc_SYMBOL = .; LONG(CHECKSUM); } } + grep -R --include="*.symversions" --exclude-dir=redhat -H -Po \ + "^__crc_\K$1 = .*(?=;)" | tr -d ' ' && return || : + grep -R --include "*.symversions" --exclude-dir=redhat -H -Po \ + "__crc_\K$1[ =].*0x[[:xdigit:]]+" \ + | sed -e 's/ = .*\(0x[[:xdigit:]]\+\).*/=\1/' + +} + +# batch call to export_tryfind, limited to 10 symbols at the time to make sure +# argv doesn't overflow +# export_tryfind SYMBOL... +batch() { + local fn=$1 + shift 1 + local o=1 + local d=10 + while [ $o -le $# ]; do + [ $o -gt $# ] && d=$((o-$#)) + # pass an array of symbols S1...Sn as a regex (S1|...|Sn) + $fn $( + echo ${@:$o:$d} \ + | sed -e 's/ /|/g' -e 's/^/(/' -e 's/$/)/' + ) + o=$((o+d)) + done | sort | uniq +} + +src_to_symversions() { + sed 's/\.[cS]$/.symversions/' +} + +src_to_symtypes() { + sed 's/\.[cS]$/.symtypes/' +} + +dump_failed_symbols() { + echo1 "Couldn't find the following symbols using this method:" >&3 + for sym in ${SYMFAIL[@]}; do + echo2 $sym >&3 + done +} + +prepare_stage() { + # SYMBOL lists symbols to be searched + # SYMFAIL lists symbols that failed in the previous run + SYMBOL=(${SYMFAIL[@]}) + SYMFAIL=() + TARGETS=() + FILE=() +} + +process_Module_symvers() { + [ ! -e Module.symvers ] && return 1 + + # read checksums into SYMCRC + local IFS=$'\n' + for sym in $(comm -12 <(cut -f2 Module.symvers | sort) \ + <(echo ${SYMBOL[@]} | tr ' ' '\n' | sort)); do + symcrc=$(grep -Po "^0x[[:xdigit:]]+(?=\t$sym\t)" Module.symvers) + file=$(grep -El -m 1 -r --include="*.symtypes" \ + --exclude-dir=redhat "^[a-zA-Z#]{,2}$sym " | head -n1) + [ -z "$file" ] && continue + SYMCRC[$sym]=$symcrc + SYMFILE_T[$sym]=$file + SYMFILE_V[$sym]=Module.symvers + [ -e "${file/.symtypes/.c}" ] && file="${file/.symtypes/.c}" + [ -e "${file/.symtypes/.S}" ] && file="${file/.symtypes/.S}" + SYMFILE[$sym]=$file + echo3 "Found $sym in $file (crc: $symcrc)," \ + "V: ${SYMFILE_V[$sym]}," \ + "T: ${SYMFILE_T[$sym]}," \ + "F: ${SYMFILE[$sym]}." >&3 + done + + return 0 +} + +process_symversions() { + # read checksums into SYMCRC and update stablelist, output format: + # filename.symversions:__crc_symbolname = 0x0000.. + for match in $(batch symbol_tryfind_symversions "$@"); do + file=${match%:*} + symcrc=${match#*:} + sym=${symcrc%%=*} + SYMCRC[$sym]=${symcrc##*=} + SYMFILE_V[$sym]=$file + SYMFILE_T[$sym]=${file/.symversions/.symtypes} + [ -e ${file/.symversions/.c} ] && file=${file/.symversions/.c} + [ -e ${file/.symversions/.S} ] && file=${file/.symversions/.S} + SYMFILE[$sym]=$file + echo3 "Found $sym in $file (crc: $symcrc)," \ + "V: ${SYMFILE_V[$sym]}," \ + "T: ${SYMFILE_T[$sym]}," \ + "F: ${SYMFILE[$sym]}." >&3 + done +} + +# generate MAKE_TARGET... +generate() { + if [ $# -eq 0 ]; then + echo "No targets given to generate." + SYMFAIL=(${SYMBOL[@]}) + return 1 + fi + + echo1 "Building symtype and symversion files ..." + for file in $@; do + echo2 "$file" >&3 + done + + # batch build at most 100 targets (symvers, symtypes files) + while [ $# -gt 0 ]; do + if [ $# -gt 100 ]; then + make ${MAKE_ARGS[@]} ${@:1:100} >&3 2>&3 || : + shift 100 + else + make ${MAKE_ARGS[@]} ${@:1:$#} >&3 2>&3 || : + shift $# + fi + done + + if ! process_Module_symvers; then + process_symversions "${SYMBOL[@]}" + fi + + echo1 "Looking for eligible symbol checksum updates ..." >&3 + for sym in ${SYMBOL[@]}; do + if [ -z "${SYMCRC[$sym]:-}" ]; then + echo3 "Couldn't find crc for $sym" >&3 + [[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym) + continue + fi + if [ ! -e "${SYMFILE[$sym]:-}" ]; then + echo3 "Couldn't find source file for $sym" >&3 + continue + fi + if [ ! -e "${SYMFILE_V[$sym]:-}" ]; then + echo3 "Couldn't find version file for $sym" >&3 + continue + fi + if [ ! -e "${SYMFILE_T[$sym]:-}" ]; then + echo3 "Couldn't find symtypes file for $sym" >&3 + continue + fi + cb_checksum $CARCH $sym ${SYMCRC[$sym]:-} ${SYMFILE[$sym]} \ + ${SYMFILE_V[$sym]} ${SYMFILE_T[$sym]} + done + + if [ -z "${SYMFAIL:-}" ]; then + cb_ready + return 0 + fi + + return 1 +} + +# process argv + +OPTIND=1 + +while getopts "TC12345hva:f:s:" opt; do + case "$opt" in + T) + CONFIG_NO_TEST=y + ;; + C) + CONFIG_NO_CLEAN=y + ;; + 1) + CONFIG_NO_METH1=y + ;; + 2) + CONFIG_NO_METH2=y + ;; + 3) + CONFIG_NO_METH3=y + ;; + 4) + CONFIG_NO_METH4=y + ;; + 5) + CONFIG_NO_METH5=y + ;; + 6) + CONFIG_NO_METH6=y + ;; + h) + usage + ;; + v) + V=1 + ;; + s) + USE_ENTIRE_STABLELIST=0 + [ -z "${CARCH:-}" ] && usage + if [ "$OPTARG" = "-" ]; then + SYMBOL+=($(cat)) + continue + fi + SYMBOL+=($OPTARG) + ;; + a) + CARCH=$OPTARG + ;; + esac +done +shift $((OPTIND-1)) + +CARCH=${CARCH:-$(uname -m)} +SOURCES=("${@:-}") + +if [ "$CARCH" = "$(uname -m)" ]; then + if ! command -v "$CPP" &> /dev/null; then + echo "ERROR: native gcc not found." + exit 1 + fi +else + if [ -z "${CROSS_COMPILE:-}" ]; then + CROSS_COMPILE="${KABI_CROSS_COMPILE_PREFIX}" + CROSS_COMPILE+="$CARCH" + CROSS_COMPILE+="${KABI_CROSS_COMPILE_SUFFIX}" + fi + + if ! [ -e "$CROSS_COMPILE$CPP" ]; then + echo "ERROR: $arch $CPP not found ($CROSS_COMPILE$CPP)" + exit 1 + fi + + MAKE_ARGS+=(ARCH=${CC_ARCH[$CARCH]} CROSS_COMPILE=$CROSS_COMPILE) +fi + +# Set up is_verbose fd +if [ ${V:-0} -gt 0 ]; then + exec 3>&1 + export PS4='$LINENO: ' + set -x +else + exec 3> /dev/null +fi + +export BASH_XTRACEFD="3" + +if [ -z "$CARCH" ]; then + err "No architecture specified." + usage +fi + +if [ ! -d "$REDHAT/kabi/kabi-module/kabi_$CARCH" ]; then + err "Architecture $CARCH not supported." + exit 1 +fi + +trap cleanup EXIT + +# --- prep + +cd "$(git rev-parse --show-toplevel)" + +REDHAT=${REDHAT:-$(pwd)/redhat/} + +if [ $USE_ENTIRE_STABLELIST -eq 1 ]; then + SYMBOL=($(find $REDHAT/kabi/kabi-module/kabi_$CARCH -type f \ + -not -name ".*" -exec basename {} \; | sort | uniq)) + if [ -z "${SYMBOL[*]}" ]; then + err "No symbols found on stablelist. Nothing to do." + exit + fi +elif [ -z "${SYMBOL[*]}" ]; then + err "No symbols given. Nothing to do." + exit +fi + +echo0 "The following symbol entries will be updated:" + +for symbol in ${SYMBOL[@]}; do + if find $REDHAT/kabi/kabi-module/${CARCH/#/kabi_} -name $symbol \ + -exec false {} + &> /dev/null; then + echo1 "$symbol (not found on ${CARCH} stablelist)" + else + echo1 "$symbol" + fi +done + +if [ -z "${CONFIG_NO_CLEAN+x}" ]; then + make -j$(nproc) V=$V mrproper >&3 2>&3 +fi + +if [ ! -e $REDHAT/configs/kernel*$CARCH.config ]; then + echo0 "Generating config files ..." + make dist-configs V=$V >&3 +fi + +cp $REDHAT/configs/kernel*$(uname -m).config .config + +echo0 "Building scripts (genksyms) ..." + +make -j$(nproc) V=$V scripts >&3 2>&3 # -> genksyms, requires native cc + +cp $REDHAT/configs/kernel*$CARCH.config .config + +if [ -z "${CONFIG_NO_TEST+x}" ]; then + TEST_FILE=$({ grep -rl EXPORT_SYMBOL net/core net/ . || : ; } | head -n1) + TEST_SYMVERSIONS="$(printf "$TEST_FILE" | src_to_symversions)" + TEST_SYMTYPES="$(printf "$TEST_FILE" | src_to_symtypes)" + echo1 "Checking that genksyms works as expected on a test file" \ + "$TEST_FILE, expecting $TEST_SYMVERSIONS, $TEST_SYMTYPES ..." >&3 + make V=$V ${MAKE_ARGS[@]} $TEST_SYMVERSIONS $TEST_SYMTYPES >&3 2>&3 + if [ ! -e $TEST_SYMVERSIONS -o ! -e $TEST_SYMTYPES ]; then + warn "An attempt to test genksyms failed. Please re-run with" \ + "-v and inspect the output and git diff to make sure" \ + "that the script works correctly." + fi +fi + +# --- + +queue_target() { + for f in "$@"; do + symv=($(printf "$f" | src_to_symversions)) + symt=($(printf "$f" | src_to_symtypes)) + + # ensure it's not queued already + if [[ ! " ${FILE[*]} " =~ " $symv " ]]; then + FILE+=($symv) + echo1 "$f: queued targets: $symv" >&3 + fi + + if [[ ! " ${FILE[*]} " =~ " $symt " ]]; then + FILE+=($symt) + echo1 "$f: queued targets: $symt" >&3 + fi + done +} + +targets_from_srclist() { + for src in "${@:-}"; do + if ! [ -e "$src" ]; then + err "File \`$src' not found!" + exit 1 + fi + queue_target "$src" + done +} + +targets_from_symlist() { + for sym in ${@:-}; do + if ! [ -e $REDHAT/kabi/kabi-module/kabi_$CARCH/$sym ]; then + echo1 "$sym not found in $CARCH stablelist, skipping" \ + "(missing: $REDHAT/kabi/kabi-module/kabi_$CARCH/$sym)" >&3 + [[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym) + continue + fi + src="$(grep -Po '^#P:\K.*' $REDHAT/kabi/kabi-module/kabi_$CARCH/$sym || :)" + if [ -z "$src" ]; then + echo1 "$sym does not reference source file, skipping" \ + "(missing: $REDHAT/kabi/kabi-module/kabi_$CARCH/$sym)" >&3 + [[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym) + continue + fi + if ! [ -e "$src" ]; then + echo1 "$sym: source $src got moved or removed, skipping" >&3 + [[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym) + continue + fi + queue_target "$src" + done +} + +targets_cscope() { + command -v cscope >&3 || return + for sym in "${@:-}"; do + queue_target $(cscope -k -d -L1$sym | cut -f1 -d' ' || :) + done +} + +targets_from_symlist_any() { + for sym in ${@:-}; do + fil=($(find $REDHAT/kabi/kabi-module/ -type f -name "$sym")) + if [ ${#fil[@]} -eq 0 ]; then + echo1 "$sym not found in any stablelist, skipping" \ + "(missing: $REDHAT/kabi/kabi-module/*/$sym)" >&3 + [[ ! " ${SYMFAIL[*]} " =~ " $sym " ]] && SYMFAIL+=($sym) + continue + fi + for f in ${fil[@]}; do + queue_target $(grep -Po '^#P:\K.*' $f | sort | uniq || :) + done + done +} + +targets_naive() { + # symbol got moved, naive greedy search for export statement + # can produce any non-negative number of results: + # #res > 1 : arch-specific code may yield multiple files in arch, + # leaving it to kbuild to fail instead of trying to + # mask them + # #res = 1 : ok, provided this is not a false positive (we can + # tell that when we generate checksums) + # #res = 0 : this may occur when the export symbol call/symbol + # name is composed using preprocessor concatenation; + # in this case we are forced to generate all checksums + for src in $(batch export_tryfind "$@"); do + queue_target "$src" + done +} + +targets_dry_run() { + queue_target $({ make ${MAKE_ARGS[@]} --dry-run 2>&3 || : ; } \ + | grep -E '(gcc|as)' \ + | grep -Po " \K[a-zA-Z0-9][^ ,.]*[^/ ,.]\.[cS]" \ + | sort \ + | uniq \ + | sed 's/^\(.*\)\.[cS]$/\1.symtypes\n\1.symversions/g') +} + +targets_compile() { + declare -A wrappers + local template=$(mktemp -t -u "kabi-wrapper-XXXXXXXXX") + local list=$(mktemp -u "kabi-list-XXXXXXXXX") + TEMP_FILES=("${TEMP_FILES[@]}" "$list") + for wrapper in $(grep -Po '= \$\(CROSS_COMPILE\)\K.*' Makefile); do + wrappers[$wrapper]=$template-$wrapper + TEMP_FILES=("${TEMP_FILES[@]}" "${wrappers[$wrapper]}") + bin=${CROSS_COMPILE:-}$wrapper list=$list envsubst '$bin $list' \ + > "${wrappers[$wrapper]}" <<-'EOF' + #!/usr/bin/env bash + echo "$*" | grep -Po " \K[a-zA-Z0-9_/-]+\.[cS]" | tr ' ' '\n' >> $list + $bin "$@" + EOF + chmod +x "${wrappers[$wrapper]}" + done + make ${MAKE_ARGS[@]} CROSS_COMPILE="$template-" >&3 2>&3 || : + # no need to build symvers, using Module.symvers instead + queue_target $(cat $list | src_to_symtypes | sort | uniq) +} + +SYMFAIL=(${SYMBOL[@]}) + +if [ -z "${CONFIG_NO_METH1+x}" -a $# -gt 0 ]; then + prepare_stage + echo0 "Updating stablelist using user-supplied files ..." + targets_from_srclist "${SOURCES[@]}" + generate "${FILE[@]}" && exit 0 || : + dump_failed_symbols +fi + +if [ -z "${CONFIG_NO_METH2+x}" ]; then + prepare_stage + echo0 "Updating stablelist using stablelist-provided files (arch specific) ..." + targets_from_symlist "${SYMBOL[@]}" + generate "${FILE[@]}" && exit 0 || : + dump_failed_symbols + + prepare_stage + echo0 "Updating stablelist using stablelist-provided files (any archs) ..." + targets_from_symlist_any "${SYMBOL[@]}" + generate "${FILE[@]}" && exit 0 || : + dump_failed_symbols +fi + +if [ -z "${CONFIG_NO_METH3+x}" ]; then + prepare_stage + echo0 "Updating stablelist using cscope-provided files ..." + targets_cscope "${SYMBOL[@]}" + generate "${FILE[@]}" && exit 0 || : + dump_failed_symbols +fi + +if [ -z "${CONFIG_NO_METH4+x}" ]; then + prepare_stage + echo0 "Updating stablelist using naive method ..." + targets_naive "${SYMBOL[@]}" + generate "${FILE[@]}" && exit 0 || : + dump_failed_symbols +fi + +if [ -z "${CONFIG_NO_METH5+x}" ]; then + prepare_stage + echo0 "Updating stablelist greedy method (this might take some time) ..." + targets_dry_run + generate "${FILE[@]}" && exit 0 || : + dump_failed_symbols +fi + +if [ -z "${CONFIG_NO_METH6+x}" ]; then + prepare_stage + echo0 "Updating stablelist by compiling the kernel (this might take some time) ..." + targets_compile + generate "${FILE[@]}" && exit 0 +fi + +err "could not update all of the symbol checksums required:" +for sym in ${SYMFAIL[@]}; do + printf "\t%s\n" "$sym" +done +exit 1 diff --git a/redhat/kabi/symtypes b/redhat/kabi/symtypes new file mode 100755 index blahblah..blahblah 100755 --- /dev/null +++ b/redhat/kabi/symtypes @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 + +import os, sys, argparse, re, difflib, json + +def jsonKeys2int(x): + try: + if isinstance(x, dict): + return {int(k):v for k,v in x.items()} + return x + except ValueError: + return x + +def symtypes_parse(path, data = None): + if not data: + data = { + 'children': { 0 : []}, + 'parents': { 0 : []}, + 'strtab': ["(root)"], + 'index' : {}, + 'file' : { } + } + + bpath = os.path.basename(path) + data["file"] = {} + data["file"][bpath] = { 0 : "" } + + with open(path, 'r') as fp_ref: + for line in fp_ref.readlines(): + lsplit = line.split(' ') + root = lsplit.pop(0) + children = list(filter(lambda x: len(x) > 2 and (x[1] == '#' or x == "UNKNOWN"), lsplit)) + if root in data["index"]: + idx = data["index"][root] + if idx in data["children"] and len(data["children"][idx]) > 0: + continue + index = data_add(data, root, 0, bpath, line) + for child in children: + child_index = data_add(data, child, index, "", "") + return data + +def data_add(data, ident, parent, bpath, line): + index = data['strtab'].index(ident) if ident in data['strtab'] else -1 + if index == -1: + index = len(data['strtab']) + data['strtab'].append(ident) + data['index'][ident] = index + data['children'][parent].append(index) + if index not in data['children']: + data['children'][index] = [] + if index in data['parents']: + if parent not in data['parents'][index]: + data['parents'][index].append(parent) + else: + data['parents'][index] = [parent] + if bpath and line: + data['file'][bpath][index] = line + return index + +#def symtypes_dfs(data_a, source, sink, trace, inverse = False): +# print(' > '.join(list(map(lambda i: data_a['strtab'][i], path + [e])))) +def symtypes_dfs(data, start, inverse=False, full=False): + start_i = data["index"][start] + stack = [(start_i,[start_i])] + visited = set() + paths = [] + while stack: + (node, path) = stack.pop() + if full and node in path[:-1]: + continue + if not full: + if node in visited: + continue + visited.add(node) + paths.append(path) + if not inverse: + for child in reversed(data["children"][node]): + stack.append((child, path + [child])) + else: + for child in reversed(data["parents"][node]): + if child == 0: + continue + stack.append((child, path + [child])) + + return visited, paths + +def st_open(path): + if not path: + raise ValueError("Blank blank.") + if not os.path.exists(path): + raise OSError(f"Path {path} does not exist.") + with open(path, "r") as f: + try: + return json.load(f, object_hook=jsonKeys2int) + except ValueError: + pass + return symtypes_parse(path) + +def st_write(path, data): + with open(path, "w+") as f: + if "file" in data: + del data["file"] + json.dump(data, f) + +def index(symtype, output): + data = st_open(symtype) + if output: + st_write(output, data) + return data + +def st_print(node): + if node[1] != '#': + return node + if node[0] == 's': + return "struct " + node[2:] + if node[0] == 't': + return "typedef " + node[2:] + if node[0] == 'E': + return "enum const " + node[2:] + if node[0] == 'e': + return "enum " + node[2:] + if node[0] == 'u': + return "union " + node[2:] + return node + +def im(file, dump_list, dump_path, dump_tree, start, inverse, silent): + data = index(file, None) + + if start not in data["index"]: + if not silent: + print(f"Node {start} not found in file {file}. Exitting.") + sys.exit(1) + + nodes, paths = symtypes_dfs(data, start, inverse) + + if dump_list: + for node in map(lambda i: data['strtab'][i], nodes): + print(f"{st_print(node)} (symtype node: {node})") + + if not dump_path and not dump_tree: + return + + for path in paths: + if dump_tree: + print((len(path)-1)*" " + " - " + f"{st_print(data['strtab'][path[-1]])} (symtype node: {data['strtab'][path[-1]]})"); + continue + if dump_path: + print(list(map(lambda i: data["strtab"][i], path))) + +def diff(ref, new, start): + data_ref = index(ref, None) + data_new = index(new, None) + + nodes_ref, _ = symtypes_dfs(data_ref, start) + nodes_new, _ = symtypes_dfs(data_new, start) + + nodes_ref_lbl = set(map(lambda i: data_ref['strtab'][i], nodes_ref)) + nodes_new_lbl = set(map(lambda i: data_new['strtab'][i], nodes_new)) + nodes_all = nodes_ref_lbl | nodes_new_lbl + nodes_13 = nodes_all - nodes_ref_lbl + nodes_23 = nodes_all - nodes_new_lbl + + if nodes_23: + print("The following nodes were encountered only in reference symtypes:") + print("\t" + "\n\t".join(nodes_23)) + + if nodes_13: + print("The following nodes were encountered only in new symtypes:") + print("\t" + "\n\t".join(nodes_13)) + + bpath_a = os.path.basename(ref) + bpath_b = os.path.basename(new) + for node in nodes_all - (nodes_13 | nodes_23): + idx_a = data_ref['index'][node] + idx_b = data_new['index'][node] + r = set(map(lambda i: data_ref['strtab'][i], data_ref['children'][idx_a])) + n = set(map(lambda i: data_new['strtab'][i], data_new['children'][idx_b])) + + if r == n: + continue + + i = ["\t"+data_ref['file'][bpath_a][idx_a]], \ + ["\t"+data_new['file'][bpath_b][idx_b]] + + if i[0] != i[1]: + print(f"Possible breakage detected for {st_print(node)} (symtype node: {node}) ...") + if len(n) == 1 and "UNKNOWN" in n: + print("\treplaced by UNKNOWN. Please inspect changes to #include directives") + if len(r) == 1 and "UNKNOWN" in r: + print("\tUNKNOWN got replaced. Please inspect changes to #include directives") + diff = difflib.ndiff(i[0], i[1]) + print(''.join(diff), end="") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers(help='Modes of operation.', + dest="mode") + parser_index = subparsers.add_parser('index', + help='Calculate symtypes index.') + parser_image = subparsers.add_parser('image', + help='Show type/symbol dependencies.') + parser_preimage = subparsers.add_parser('preimage', + help='Show type/symbol preimage.') + parser_df = subparsers.add_parser('diff', + help='Calculate simple symtype diff.') + + parser_index.add_argument('-o', '--output', type=str, required=True, + help='Output index file.') + parser_index.add_argument('symtype', type=str) + + for p in [ parser_image, parser_preimage ]: + p.add_argument('-i', '--index', action='store_true', + help="Input is an index file.") + p.add_argument('-S', '--silent', action='store_true') + p.add_argument('-l', '--ls', action='store_true', + help="List dependent nodes.") + p.add_argument('-p', '--path', action='store_true', + help="List paths to dependent nodes.") + p.add_argument('-t', '--tree', action='store_true', + help="Dump tree.") + p.add_argument('-s', '--start', type=str, nargs='?', + help="Start symtype entry/entries.") + p.add_argument('symtype', type=str) + + parser_df.add_argument('reference', type=str) + parser_df.add_argument('new', type=str) + parser_df.add_argument('-s', '--start', type=str, nargs='?', + help="Start symtype entry/entries.") + + args = parser.parse_args() + + if args.mode == "index": + index(args.symtype, args.output) + elif args.mode == "image" or args.mode == "preimage": + im(args.symtype, args.ls, args.path, args.tree, args.start, args.mode == "preimage", args.silent) + elif args.mode == "diff": + diff(args.reference, args.new, args.start) diff --git a/redhat/kabi/update-kabi b/redhat/kabi/update-kabi new file mode 100755 index blahblah..blahblah 100755 --- /dev/null +++ b/redhat/kabi/update-kabi @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +usage_desc() { + sed 's/^\t//' <<EOF + Updates the kernel ABI stablelist checksum information and symtypes + for architecture ARCH. + + The file updates stablelist definition in \$REDHAT/kabi/*. It does not + stage the newly updated stablelist. +EOF +} + +emit_line_from_Module_symvers() { + [ ! -e Module.symvers ] && return 1 + grep -P "\t$2\t" Module.symvers +} + +emit_line() { + local checksum="$1" + local symbol="$2" + local obj="${3:-unknown}" + local macro="${4:-UNKNOWN_MACRO}" + local namespace="${5:-}" + + if [ -z "${namespace:-}" ]; then + printf "%s\t%s\t%s\t%s\n" $checksum $symbol $obj $macro + return + fi + + printf "%s\t%s\t%s\t%s\t%s\n" $checksum $symbol $obj $macro $namespace +} + +cb_checksum_new() { + local arch="$1" + local symbol="$2" + local checksum="$3" + local file="$4" + local symversions="$5" + local symtypes="$6" + local stable_entry=$REDHAT/kabi/kabi-module/kabi_$arch/$symbol + local prev_csum="NULL" + + { + printf "#0-\n" + printf "#I:0\n" + printf "#P:$file\n" + if ! emit_line_from_Module_symvers $arch $symbol; then + [ ${TEST:-0} -eq 0 ] && return 1 + emit_line $checksum $symbol null null + fi + } > $stable_entry + + echo2 "Added symbol \`$sym' for architecture \`$arch' ($checksum)." + emit_line_from_Module_symvers $arch $symbol || : +} + +cb_checksum_update() { + local arch="$1" + local symbol="$2" + local checksum="$3" + local file="$4" + local symversions="$5" + local symtypes="$6" + + local prev_index="$7" + local prev_csum="$8" + #prev_symbol $9 unused + local prev_obj="${10}" + local prev_macro="${11}" + local prev_ns="${12:-}" + + local stable_entry=$REDHAT/kabi/kabi-module/kabi_$arch/$symbol + local index=$(grep -v '^#' $stable_entry | wc -l) + + if [ "$prev_csum" = "$checksum" ]; then + echo2 "Symbol checksum of \`$sym' for architecture \`$arch' unchanged." + return + fi + + sed -i -e "s#\#P:.*#\#P:$file#" -e "s/#I:[0-9]\+/#I:$index/" \ + $stable_entry + if ! emit_line_from_Module_symvers $arch $symbol; then + emit_line $checksum $symbol $prev_obj $prev_macro $prev_ns + fi >> $stable_entry + + echo2 "Updated symbol \`$sym' for architecture \`$arch'" \ + "($prev_csum -> $checksum)." +} + +# Called whenever a new symbol checksum is obtained +# cb_checksum ARCH SYMBOL CHECKSUM FILE SYMVERSIONS_FILE SYMTYPES_FILE +cb_checksum() { + local arch="$1" + local symbol="$2" + local checksum="$3" + local symversions="$5" + local symtypes="$6" + local stable_entry=$REDHAT/kabi/kabi-module/kabi_$arch/$symbol + + cp $symtypes $REDHAT/kabi/kabi-module/kabi_$arch/.$symbol + + if [ -e $stable_entry ]; then + prev_index=$(grep -Po "^#I:\K[0-9]+" $stable_entry) + cb_checksum_update "$@" $prev_index $( + IFS=' ' grep -v "^#" $stable_entry \ + | head -n$((${prev_index}+1)) \ + | tail -n1 + ) + return + fi + + cb_checksum_new "$@" +} + +# Called whenever the generate command finished successfully +cb_ready() { + # no more symbols, nothing to do + : +} + +cd "$(git rev-parse --show-toplevel)" +REDHAT=${REDHAT:-$(pwd)/redhat/} + +# Generate new symbol checksums and symvers files +. $REDHAT/kabi/symtype-generate -- https://gitlab.com/cki-project/kernel-ark/-/merge_requests/2021 _______________________________________________ kernel mailing list -- kernel@lists.fedoraproject.org To unsubscribe send an email to kernel-le...@lists.fedoraproject.org Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/ List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedoraproject.org/archives/list/kernel@lists.fedoraproject.org Do not reply to spam, report it: https://pagure.io/fedora-infrastructure/new_issue