[Valgrind-developers] [valgrind] ltp tests: Use new LTP 20250530, allow for filtering valgrind output

[Mark W. <ma...@so...>]

https://sourceware.org/git/gitweb.cgi?p=valgrind.git;h=b8891a4c5d139b8c327ee20c37066d601fc56b0a

commit b8891a4c5d139b8c327ee20c37066d601fc56b0a
Author: Martin Cermak <mc...@re...>
Date:   Tue Jun 3 13:43:18 2025 +0200

    ltp tests: Use new LTP 20250530, allow for filtering valgrind output
    
    Use fresh LTP 20250530.
    
    Allow for filtering valgrind output:
    
    In some cases the LTP tests intentionally work with SIGSEGV.  This
    happens e.g. with the mmap18 and select03 testcases.  Valgrind
    detects SIGSEGV and reports that as a failure.
    
    Such report can't be suppressed using the suppressions mechanism.
    That's why this update comes with "output filters". Filters are
    scripts that read from their stdin, and write filtered output to
    their stdout. Filters reside in auxprogs/filters.
    
    This update comes with 2 filters: For mmap18, and for select03.
    They are awk scripts.
    
    Except for filters, this update also blacklists testcase fork13
    because it is slow.  It is possible to add comments prefixed with
    the '#' sign (implicitly - because they don't match any testcase
    name), so a comment is added too.
    
    This update also introduces new default valgrind --vgdb=no switch. It
    improves valgrind behavior with nftw01 nftw6401 setfsgid04 setfsgid03_16
    and symlink03 testcases. These were previously complaining like this:
    
    ==22969== could not unlink /tmp/vgdb-pipe-from-vgdb-to-22969-by-root-on ...

Diff:
---
 auxprogs/Makefile.am      |  4 ++--
 auxprogs/filters/mmap18   | 29 +++++++++++++++++++++++++++++
 auxprogs/filters/select03 | 18 ++++++++++++++++++
 auxprogs/ltp-excludes.txt |  2 ++
 auxprogs/ltp-tester.sh    | 46 ++++++++++++++++++++++++++--------------------
 5 files changed, 77 insertions(+), 22 deletions(-)

diff --git a/auxprogs/Makefile.am b/auxprogs/Makefile.am
index 4a10fb1cf8..97eb9501eb 100644
--- a/auxprogs/Makefile.am
+++ b/auxprogs/Makefile.am
@@ -155,8 +155,8 @@ endif
 endif
 
 # Linux Test Project
-LTP_VERSION=20250130
-LTP_SHA256_SUM=02e4ec326be54c3fd92968229a468c02c665d168a8a673edc38a891f7395ae10
+LTP_VERSION=20250530
+LTP_SHA256_SUM=27586ba78eac1e40cd422add2842f1ad70f09fea55da3bd6a25e10feb786d4f2
 LTP_TAR_NAME=ltp-full-$(LTP_VERSION).tar.xz
 LTP_URL=https://github.com/linux-test-project/ltp/releases/download/$(LTP_VERSION)/$(LTP_TAR_NAME)
 LTP_TAR=$(AUX_CHECK_DIR)/$(LTP_TAR_NAME)
diff --git a/auxprogs/filters/mmap18 b/auxprogs/filters/mmap18
new file mode 100755
index 0000000000..edaf9c15d0
--- /dev/null
+++ b/auxprogs/filters/mmap18
@@ -0,0 +1,29 @@
+#!/bin/awk -f
+
+# Filter out stuff like the following, since it is expected output for the select03 testcase:
+
+# mmap18: unempty log2.filtered:
+# ==24613== 
+# ==24613== Process terminating with default action of signal 11 (SIGSEGV): dumping core
+# ==24613==  Access not within mapped region at address 0x4B3AFF8
+# ==24613==    at 0x401F86: check_depth_recursive (mmap18.c:118)
+# ==24613==  If you believe this happened as a result of a stack
+# ==24613==  overflow in your program's main thread (unlikely but
+# ==24613==  possible), you can try to increase the size of the
+# ==24613==  main thread stack using the --main-stacksize= flag.
+# ==24613==  The main thread stack size used in this run was 8388608.
+# ==24620== 
+# ==24620== Process terminating with default action of signal 11 (SIGSEGV): dumping core
+# ==24620==  Access not within mapped region at address 0x4B2EFF8
+# ==24620==    at 0x401F86: check_depth_recursive (mmap18.c:118)
+# ==24620==  If you believe this happened as a result of a stack
+# ==24620==  overflow in your program's main thread (unlikely but
+# ==24620==  possible), you can try to increase the size of the
+# ==24620==  main thread stack using the --main-stacksize= flag.
+# ==24620==  The main thread stack size used in this run was 8388608.
+
+skip = 0
+/==[0-9][0-9]*==/ { skip = 1 }
+/Process terminating with default action of signal 11/ { skip = 1; skipblock=1 }
+/The main thread stack size used in this run was/ { skip = 1; skipblock=0 }
+!skip && !skipblock { print }
diff --git a/auxprogs/filters/select03 b/auxprogs/filters/select03
new file mode 100755
index 0000000000..368a884e7c
--- /dev/null
+++ b/auxprogs/filters/select03
@@ -0,0 +1,18 @@
+#!/bin/awk -f
+
+# Filter out stuff like the following, since it is expected output for the select03 testcase:
+
+# ==22396== 
+# ==22396== Process terminating with default action of signal 11 (SIGSEGV): dumping core
+# ==22396==  Bad permissions for mapped region at address 0x483B000
+# ==22396==    at 0x4946397: select (in /usr/lib64/libc.so.6)
+# ==22396==    by 0x4020BA: run (select_var.h:26)
+# ==22396==    by 0x40B30C: fork_testrun (tst_test.c:1566)
+# ==22396==    by 0x40D4EF: tst_run_tcases (tst_test.c:1918)
+# ==22396==    by 0x401D4D: main (tst_test.h:725)
+
+skip = 0
+/==[0-9][0-9]*==/ { skip = 1 }
+/Process terminating with default action of signal 11/ { skip = 1; skipblock=1 }
+/by.*main.*tst_test.h/ { skip = 1; skipblock=0 }
+!skip && !skipblock { print }
diff --git a/auxprogs/ltp-excludes.txt b/auxprogs/ltp-excludes.txt
index 236d779426..275fd74854 100644
--- a/auxprogs/ltp-excludes.txt
+++ b/auxprogs/ltp-excludes.txt
@@ -1,5 +1,7 @@
+# Exclude the following syscall tests because they are too slow:
 bind06
 epoll-ltp
+fork13
 fork14
 futex_cmp_requeue01
 futex_cmp_requeue02
diff --git a/auxprogs/ltp-tester.sh b/auxprogs/ltp-tester.sh
index 54d807b0c2..ba8fd8be47 100755
--- a/auxprogs/ltp-tester.sh
+++ b/auxprogs/ltp-tester.sh
@@ -11,7 +11,9 @@ ORIG_PATH=$PATH
 SCRIPT_SRC=$(dirname $0)
 LOGDIR=${LOGDIR:-$LTP_SRC_DIR/ltp/tests}
 DIFFCMD="diff -u"
-VALGRIND="${VALGRIND:-$LTP_SRC_DIR/../../../vg-in-place}"
+# vgdb sensitive testcase: e.g. nftw01 nftw6401 setfsgid04 setfsgid03_16 symlink03
+VGARGS="-q --vgdb=no"
+VALGRIND="${VALGRIND:-$LTP_SRC_DIR/../../../vg-in-place} ${VGARGS}"
 # For parallel testing, consider IO intensive jobs, take nproc into account
 PARALLEL_JOBS=${PARALLEL_JOBS:-$(nproc)}
 # TESTS env var may be specified to restrict testing to selected test cases
@@ -37,24 +39,27 @@ doTest ()
     pushd $dir >/dev/null
         PATH="$ORIG_PATH:$PWD"
         ./$exe >$l/log1std 2>$l/log1err ||:
-        $VALGRIND -q --tool=none --log-file=$l/log2 ./$exe >$l/log2std 2>$l/log2err ||:
-        $VALGRIND -q --tool=memcheck --log-file=$l/log3 ./$exe >$l/log3std 2>$l/log3err ||:
-
-        # We want to make sure that LTP syscall tests give identical
-        # results with and without valgrind.  The test logs go to the
-        # stderr.  They aren't identical across individual runs. The
-        # differences include port numbers, temporary files, test
-        # output ordering changes and more. They aren't trivially
-        # comparable.  We resort to comparing at least the final
-        # summary of individual test results
-        tail -10 $l/log1err | grep -E "^(passed|failed|broken|skipped|warnings)" > $l/log1summary ||:
-        tail -10 $l/log2err | grep -E "^(passed|failed|broken|skipped|warnings)" > $l/log2summary ||:
-        tail -10 $l/log3err | grep -E "^(passed|failed|broken|skipped|warnings)" > $l/log3summary ||:
+        $VALGRIND --tool=none --log-file=$l/log2 ./$exe >$l/log2std 2>$l/log2err ||:
+        $VALGRIND --tool=memcheck --log-file=$l/log3 ./$exe >$l/log3std 2>$l/log3err ||:
+
+        for i in "$l"/log{1std,1err,2,2std,2err,3,3std,3err}; do
+            echo "# cat $(basename $i)" >> $LOGDIR/$exe.log
+            cat $i >> $LOGDIR/$exe.log
+        done
+
+        echo "# errors" >> $LOGDIR/$exe.log
+
+        # If there is a logfile filter, apply it
+        if test -f "${SCRIPT_SRC}/filters/${exe}"; then
+            cat $l/log2 | ${SCRIPT_SRC}/filters/${exe} > $l/log2.filtered
+        else
+            cat $l/log2 > $l/log2.filtered
+        fi
 
         # Check logs, report errors
         pushd $l >/dev/null
-            if test -s log2; then
-                echo -e "${exe}: unempty log2:\n$(cat log2)" | tee -a $LOGDIR/$exe.log
+            if test -s log2.filtered; then
+                echo -e "${exe}: unempty log2.filtered:\n$(cat log2.filtered)" | tee -a $LOGDIR/$exe.log
                 rv="FAIL"
             fi
 
@@ -63,17 +68,18 @@ doTest ()
                 rv="FAIL"
             fi
 
-            if ! ${DIFFCMD} log1summary log2summary >/dev/null; then
-                echo -e "${exe}: ${DIFFCMD} log1summary log2summary:\n$(${DIFFCMD} log1summary log2summary)" | tee -a $LOGDIR/$exe.log
+            if ! ${DIFFCMD} log1err log2err >/dev/null; then
+                echo -e "${exe}: ${DIFFCMD} log1err log2err:\n$(${DIFFCMD} log1err log2err)" | tee -a $LOGDIR/$exe.log
                 rv="FAIL"
             fi
 
-            if ! ${DIFFCMD} log2summary log3summary >/dev/null; then
-                echo -e "${exe}: ${DIFFCMD} log2summary log3summary:\n$(${DIFFCMD} log2summary log3summary)" | tee -a $LOGDIR/$exe.log
+            if ! ${DIFFCMD} log2err log3err >/dev/null; then
+                echo -e "${exe}: ${DIFFCMD} log2err log3err:\n$(${DIFFCMD} log2err log3err)" | tee -a $LOGDIR/$exe.log
                 rv="FAIL"
             fi
 
             # synthetize automake style testlogs for bunsen import
+            echo "# result" >> $LOGDIR/$exe.log
             echo ":test-result: $rv" | tee -a $LOGDIR/$exe.log > $LOGDIR/$exe.trs
         popd >/dev/null
     popd >/dev/null