Menu
▾
▴
[Valgrind-developers] [valgrind] Use LTP for testing valgrind
|
From: Mark W. <ma...@so...> - 2025-04-17 20:42:43
|
https://sourceware.org/git/gitweb.cgi?p=valgrind.git;h=944a8c864c5e1df6388bf4b47900eb9c001ceb73 commit 944a8c864c5e1df6388bf4b47900eb9c001ceb73 Author: Martin Cermak <mc...@re...> Date: Thu Apr 17 16:14:19 2025 +0200 Use LTP for testing valgrind Add a new top level make target ltpchecks which will fetch the latest linux test project (ltp) release as defined by the LTP_VERSION and LTP_SHA256 variables in auxprogs/Makefile.am (update those when a new version of ltp is released). If the ltp tar.xz has already been downloaded, or it has already been unpacked and build, the (cached) file and build will be reused. The actual testing is done through the auxprogs/ltp-tester.sh script. It takes all executable tests from the ltp testcases under kernel/syscalls and runs them 3 times. Once directly, not under valgrind, once with -q --tool=none and once with -q --tool=memcheck. It then checks that valgrind didn't produce any messages with the none tool, that there were no fatal errors produced (as defined in auxprogs/ltp-error-patterns.txt) and that the ltp results are the same with and without valgrind. Currently there are 1472 test binaries and running them all (serially) takes more than three hours and detects various missing or incomplete syscall handlers in valgrind, plus various crashers. https://bugs.kde.org/show_bug.cgi?id=502679 Diff: --- Makefile.am | 3 ++ NEWS | 1 + auxprogs/Makefile.am | 35 +++++++++++++++++- auxprogs/ltp-error-patterns.txt | 6 ++++ auxprogs/ltp-tester.sh | 78 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 122 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 14309dacfa..e67356b5a0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -103,6 +103,9 @@ perf: check auxchecks: all $(MAKE) -C auxprogs auxchecks +ltpchecks: all + $(MAKE) -C auxprogs ltpchecks + # Nb: no need to include any Makefile.am files here, or files included from # them, as automake includes them automatically. Also not COPYING, README # or NEWS. diff --git a/NEWS b/NEWS index fac28e7984..e77c62456b 100644 --- a/NEWS +++ b/NEWS @@ -76,6 +76,7 @@ are not entered into bugzilla tend to get forgotten about or ignored. 502126 glibc 2.41 extra syscall_cancel frames 502288 s390x: Memcheck false positives with NNPA last tensor dimension 502324 s390x: Memcheck false positives with TMxx and TM/TMY +502679 Use LTP for testing valgrind To see details of a given bug, visit https://bugs.kde.org/show_bug.cgi?id=XXXXXX diff --git a/auxprogs/Makefile.am b/auxprogs/Makefile.am index 3a9709da6f..00c743bc11 100644 --- a/auxprogs/Makefile.am +++ b/auxprogs/Makefile.am @@ -124,6 +124,9 @@ endif auxchecks: gsl-check auxclean: gsl-clean +ltpchecks: ltp-check +ltpclean: ltp-clean + # You can override AUX_CHECK_DIR to use a previous download/build. # Must be an absolute path. # e.g. make auxchecks AUX_CHECK_DIR=$HOME/valgrind-auxtests @@ -148,6 +151,14 @@ GSL_CFLAGS+=-mfpmath=sse -msse2 endif endif +# Linux Test Project +LTP_VERSION=20250130 +LTP_SHA256_SUM=02e4ec326be54c3fd92968229a468c02c665d168a8a673edc38a891f7395ae10 +LTP_TAR_NAME=ltp-full-$(LTP_VERSION).tar.xz +LTP_URL=https://github.com/linux-test-project/ltp/releases/download/$(LTP_VERSION)/$(LTP_TAR_NAME) +LTP_TAR=$(AUX_CHECK_DIR)/$(LTP_TAR_NAME) +LTP_SRC_DIR=$(AUX_CHECK_DIR)/ltp-full-$(LTP_VERSION) + # Trick to get a literal space to use in substitutions sp := $(subst ,, ) @@ -166,6 +177,10 @@ $(GSL_TAR): mkdir -p $(AUX_CHECK_DIR) wget -q -O $(GSL_TAR) $(GSL_URL) +$(LTP_TAR): + mkdir -p $(AUX_CHECK_DIR) + wget -q -O $(LTP_TAR) $(LTP_URL) + # We need to autoreconf to make sure to get config.guess, config.sub # and libtool for newer architectures. $(GSL_SRC_DIR)/gsl-patched: $(GSL_TAR) @@ -177,6 +192,15 @@ $(GSL_SRC_DIR)/gsl-patched: $(GSL_TAR) autoreconf -f -i -Wnone) touch $@ +$(LTP_SRC_DIR): $(LTP_TAR) + echo "$(LTP_SHA256_SUM) $(LTP_TAR)" | @SHA256SUM@ --check - + (cd $(AUX_CHECK_DIR) && \ + tar Jxf $(LTP_TAR_NAME) && \ + cd $(LTP_SRC_DIR) && \ + ./configure CC="${CC}" CXX="${CXX}" CFLAGS="$(LTP_CFLAGS)" && \ + ${MAKE} -j $(nproc) -C testcases/kernel/syscalls) + touch $@ + # We need make check -k because # some tests might fail even native (only on i386 though). # make check doesn't work reliably with -j. @@ -204,17 +228,26 @@ gsl-check: $(GSL_BUILD_DIR)/gsl-build diff -u $(abs_top_srcdir)/auxprogs/gsl-1.6.out.x86.exp \ $(GSL_BUILD_DIR)/valgrind-gsl.out +ltp-check: $(LTP_SRC_DIR) + LTP_SRC_DIR=$(LTP_SRC_DIR) \ + VALGRIND=$(abs_top_builddir)/vg-in-place \ + $(abs_top_srcdir)/auxprogs/ltp-tester.sh + + # We keep the tarball but remove the unpacked sources and build gsl-clean: rm -rf $(GSL_SRC_NAME) $(GSL_BUILD_DIR) +ltp-clean: + rm -rf $(LTP_SRC_DIR) + #---------------------------------------------------------------------------- # General stuff #---------------------------------------------------------------------------- all-local: inplace-noinst_PROGRAMS inplace-noinst_DSYMS -clean-local: clean-noinst_DSYMS auxclean +clean-local: clean-noinst_DSYMS auxclean ltpclean install-exec-local: install-noinst_PROGRAMS install-noinst_DSYMS diff --git a/auxprogs/ltp-error-patterns.txt b/auxprogs/ltp-error-patterns.txt new file mode 100644 index 0000000000..ee5048bf42 --- /dev/null +++ b/auxprogs/ltp-error-patterns.txt @@ -0,0 +1,6 @@ +VALGRIND INTERNAL ERROR +^valgrind: +WARNING: unhandled +unhandled +Valgrind does not support +Valgrind abort diff --git a/auxprogs/ltp-tester.sh b/auxprogs/ltp-tester.sh new file mode 100755 index 0000000000..a3c2157c56 --- /dev/null +++ b/auxprogs/ltp-tester.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +set -e + +if [ -z "${LTP_SRC_DIR:-}" ]; then + echo "ERROR: LTP_SRC_DIR needs to be set. Dying, bye bye ..." + exit 1 +fi + +ORIG_PATH=$PATH +ORIG_PWD=$PWD +LOGDIR=${LOGDIR:-$LTP_SRC_DIR/valgrind-ltp-logs} +SUMMARY_LOG="$LOGDIR/summary.log" +DIFFCMD="diff -u" +VALGRIND="${VALGRIND:-$LTP_SRC_DIR/../../../vg-in-place}" + +# Initialize LOGDIR +mkdir -p $LOGDIR; rm -rf $LOGDIR/* + +myLog () +{ + msg="$1" + echo "$msg" + echo -e "FAIL: $msg" >> $SUMMARY_LOG +} + +cd $LTP_SRC_DIR + +mapfile -t files < <(find testcases/kernel/syscalls -executable -and -type f | sort) +c=${#files[@]}; i=0 + +for test in "${files[@]}"; do + dir=$(dirname $test) + exe=$(basename $test) + l="$LOGDIR/$exe" + mkdir -p $l + i=$((++i)) + pushd $dir >/dev/null + echo "[$i/$c] Testing $exe ..." | tee -a $SUMMARY_LOG + PATH="$ORIG_PATH:$PWD" + ./$exe >$l/log1std 2>$l/log1err ||: + $VALGRIND -q --tool=none --log-file=$l/log2 ./$exe >$l/log2std 2>$l/log2err ||: + $VALGRIND -q --tool=memcheck --log-file=$l/log3 ./$exe >$l/log3std 2>$l/log3err ||: + + # We want to make sure that LTP syscall tests give identical + # results with and without valgrind. The test logs go to the + # stderr. They aren't identical across individual runs. The + # differences include port numbers, temporary files, test + # output ordering changes and more. They aren't trivially + # comparable. We resort to comparing at least the final + # summary of individual test results + tail -10 $l/log1err | grep -E "^(passed|failed|broken|skipped|warnings):" > $l/log1summary ||: + tail -10 $l/log2err | grep -E "^(passed|failed|broken|skipped|warnings):" > $l/log2summary ||: + tail -10 $l/log3err | grep -E "^(passed|failed|broken|skipped|warnings):" > $l/log3summary ||: + + # Check logs, report errors + pushd $l >/dev/null + if test -s log2; then + myLog "${exe}: unempty log2:\n$(cat log2)" + fi + + if grep -f $ORIG_PWD/ltp-error-patterns.txt * > error-patterns-found.txt; then + myLog "${exe}: error string found:\n$(cat error-patterns-found.txt)" + fi + + if ! ${DIFFCMD} log1summary log2summary >/dev/null; then + myLog "${exe}: ${DIFFCMD} log1summary log2summary:\n$(${DIFFCMD} log1summary log2summary)" + fi + + if ! ${DIFFCMD} log2summary log3summary >/dev/null; then + myLog "${exe}: ${DIFFCMD} log2summary log3summary:\n$(${DIFFCMD} log2summary log3summary)" + fi + popd >/dev/null + popd >/dev/null +done + +echo "TESTING FINISHED, logs in $LOGDIR" + |
