Fix profiler and TAU harness checks

HIP/jacobi/Makefile

@@ -27,7 +27,10 @@ ifneq ($(USE_CRAYPE),)
   MPICC=CC
 endif
 
-IS_OMPI=$(shell which ompi_info 2>/dev/null)
+# Detect OpenMPI by asking the wrapper itself: only OpenMPI's wrapper answers
+# --showme:version. This is more reliable than checking PATH when more than one
+# MPI is installed.
+IS_OMPI=$(shell ${MPICC} --showme:version >/dev/null 2>&1 && echo openmpi)
 # MPICH detection. The Cray cc/CC wrapper prints "CrayPE is loaded" (matched by
 # 'Cray' below). The standalone mpich-wrappers MPICH on a Cray, however, reports
 # the underlying compiler from `--version` (e.g. "AMD clang"), so grepping the
@@ -58,6 +61,13 @@ else
     $(error Unknown MPI version! Currently can detect mpich or openmpi)
 endif
 
+# The link is done by hipcc, so MPI reaches it only through MPILDFLAGS. If those
+# came back empty, stop now with the likely cause instead of failing later with
+# undefined MPI symbols.
+ifeq ($(strip $(MPILDFLAGS)),)
+    $(error MPI detected but $(strip ${MPICC}) returned no link flags. Two MPIs on PATH is the usual cause: check that mpic++ and any MPI-linked module you loaded are the same MPI)
+endif
+
 # Flags
 CFLAGS=-O3 -g -ggdb -fPIC -std=c++11 -march=native -Wall
 CFLAGS+=-I$(ROCM_PATH)/roctracer/include -I${ROCM_PATH}/include/roctracer

MLExamples/PyTorch_Profiling/rocm-compute-profiler/single_process.sh

@@ -7,6 +7,39 @@ PROFILER_TOP_DIR="$(dirname "$(dirname "$(readlink -fm "$0")")")"
 # Call the software set up script:
 source ${PROFILER_TOP_DIR}/setup.sh
 
+# Dependency precondition. `analyze` enforces the exact pins in its own
+# requirements.txt and this run ends in `analyze`, so verify them before
+# spending around 25 minutes profiling. Fail fast rather than time out.
+_rpc_bin="$(command -v rocprof-compute 2>/dev/null)"
+if [ -n "${_rpc_bin}" ]; then
+  _rpc_req="$(dirname "$(dirname "${_rpc_bin}")")/libexec/rocprofiler-compute/requirements.txt"
+  if [ -f "${_rpc_req}" ] && ! python3 - "${_rpc_req}" <<'PINS'
+import sys
+from importlib.metadata import version, PackageNotFoundError
+bad = []
+for raw in open(sys.argv[1]):
+    raw = raw.split('#')[0].strip()
+    if not raw or '==' not in raw:
+        continue
+    name, want = raw.split('==', 1)
+    try:
+        have = version(name)
+    except PackageNotFoundError:
+        bad.append('%s: absent (needs %s)' % (name, want)); continue
+    if have != want:
+        bad.append('%s: %s installed, needs %s' % (name, have, want))
+for b in bad:
+    print('  ' + b)
+sys.exit(1 if bad else 0)
+PINS
+  then
+    echo "rocprof-compute dependency pins are unsatisfied under $(python3 -V 2>&1)."
+    echo "'analyze' cannot run, so skipping the profile run instead of timing out."
+    echo "See ${_rpc_req} for the required versions."
+    exit 1
+  fi
+fi
+
 export NPROCS=1
 
 pushd ${PROFILER_TOP_DIR}

tests/CMakeLists.txt

@@ -1696,11 +1696,11 @@ set_property(TEST Rocprof-sys_ROCm_Run_Check PROPERTY PASS_REGULAR_EXPRESSION "T
 set_property(TEST Rocprof-sys_ROCm_Run_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider")
 
 add_test(NAME Rocprof-sys_ROCm_Stream_Overlap COMMAND ../rocprof-sys_stream_overlap.sh )
-set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY PASS_REGULAR_EXPRESSION "proto")
+set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY PASS_REGULAR_EXPRESSION "ROCPROFSYS_STREAM_OVERLAP_RESULT: PASS")
 set_property(TEST Rocprof-sys_ROCm_Stream_Overlap PROPERTY SKIP_REGULAR_EXPRESSION "module spider")
 
 add_test(NAME Rocprof-compute_ROCm_Roofline_Check COMMAND ../rocprof-compute_roofline_check.sh)
-set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY PASS_REGULAR_EXPRESSION "Empirical Roofline .* saved!")
+set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY PASS_REGULAR_EXPRESSION "ROOFLINE_RESULT: PASS")
 set_property(TEST Rocprof-compute_ROCm_Roofline_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider" "disabled on MI300")
 set_tests_properties(Rocprof-compute_ROCm_Roofline_Check PROPERTIES TIMEOUT 600)
 
@@ -1756,19 +1756,19 @@ set_property(TEST JAX_Mnist PROPERTY SKIP_REGULAR_EXPRESSION "module spider" "Un
 # TAU Tests
 
 add_test(NAME TAU_Trace_Check COMMAND ../tau_exec_check.sh --tau-trace)
-set_property(TEST TAU_Trace_Check PROPERTY PASS_REGULAR_EXPRESSION "tautrace.0")
+set_property(TEST TAU_Trace_Check PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: trace artifact PRESENT")
 set_property(TEST TAU_Trace_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'")
 
 add_test(NAME TAU_Profile_Check COMMAND ../tau_exec_check.sh --tau-profile)
-set_property(TEST TAU_Profile_Check PROPERTY PASS_REGULAR_EXPRESSION "profile.0")
+set_property(TEST TAU_Profile_Check PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: profile artifact PRESENT")
 set_property(TEST TAU_Profile_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'")
 
 add_test(NAME TAU_Check_HIP_Profile COMMAND ../tau_exec_check.sh --tau-profile)
-set_property(TEST TAU_Check_HIP_Profile PROPERTY PASS_REGULAR_EXPRESSION "hipMemcpy")
+set_property(TEST TAU_Check_HIP_Profile PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: HIP routines PRESENT in profile")
 set_property(TEST TAU_Check_HIP_Profile PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'")
 
 add_test(NAME TAU_Check_MPI_Profile COMMAND ../tau_exec_check.sh --tau-profile)
-set_property(TEST TAU_Check_MPI_Profile PROPERTY PASS_REGULAR_EXPRESSION "MPI_Allreduce()")
+set_property(TEST TAU_Check_MPI_Profile PROPERTY PASS_REGULAR_EXPRESSION "TAU_CHECK: MPI routines PRESENT in profile")
 set_property(TEST TAU_Check_MPI_Profile PROPERTY SKIP_REGULAR_EXPRESSION "module spider tau" "Unable to locate a modulefile for 'tau'")
 set_property(TEST TAU_Check_MPI_Profile PROPERTY TIMEOUT 600)
 

tests/rocprof-compute_analyze_check.sh

@@ -22,5 +22,14 @@ cmake ..
 make
 
 export HSA_XNACK=1
+# rocprof-compute needs Python >= 3.10: its native_tool_finder.py annotates with
+# a PEP 604 union (Path | None) and has no "from __future__ import annotations",
+# so on Python 3.9 the annotation raises TypeError at import. Fail here with the
+# reason rather than later from inside the tool.
+if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then
+  echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)."
+  echo "ERROR: load a newer Python before running this test."
+  exit 1
+fi
 rocprof-compute profile -n v1 --no-roof -- ./saxpy
 rocprof-compute analyze -p workloads/v1/* --block 7.1.0 7.1.1 7.1.2 7.1.0: Grid size 7.1.1: Workgroup size 7.1.2: Total Wavefronts

tests/rocprof-compute_profile_check.sh

@@ -23,4 +23,13 @@ cmake ..
 make
 
 export HSA_XNACK=1
+# rocprof-compute needs Python >= 3.10: its native_tool_finder.py annotates with
+# a PEP 604 union (Path | None) and has no "from __future__ import annotations",
+# so on Python 3.9 the annotation raises TypeError at import. Fail here with the
+# reason rather than later from inside the tool.
+if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then
+  echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)."
+  echo "ERROR: load a newer Python before running this test."
+  exit 1
+fi
 rocprof-compute profile -n v1 --no-roof -- ./saxpy

tests/rocprof-compute_roofline_check.sh

@@ -30,4 +30,22 @@ cmake ..
 make
 
 export HSA_XNACK=1
+# rocprof-compute needs Python >= 3.10: its native_tool_finder.py annotates with
+# a PEP 604 union (Path | None) and has no "from __future__ import annotations",
+# so on Python 3.9 the annotation raises TypeError at import. Fail here with the
+# reason rather than later from inside the tool.
+if ! python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3,10) else 1)' 2>/dev/null; then
+  echo "ERROR: rocprof-compute needs Python >= 3.10, but python3 is $(python3 -V 2>&1)."
+  echo "ERROR: load a newer Python before running this test."
+  exit 1
+fi
 rocprof-compute profile -n rooflines_PDF --roof-only  -- ./saxpy
+
+# Assert the roofline artifact exists rather than matching the tool's log
+# wording, which changed when omniperf became rocprofiler-compute.
+_roofline_csv="$(find . -name roofline.csv -size +0 2>/dev/null | head -1)"
+if [ -n "${_roofline_csv}" ]; then
+  echo "ROOFLINE_RESULT: PASS roofline data at ${_roofline_csv}"
+else
+  echo "ROOFLINE_RESULT: FAIL no non-empty roofline.csv produced"
+fi

tests/rocprof-sys_stream_overlap.sh

@@ -126,9 +126,27 @@ ${TOOL_COMMAND}-avail -G $PWD/.configure.cfg
 export ${TOOL_CONFIG}_CONFIG_FILE=$PWD/.configure.cfg
 ${TOOL_COMMAND}-instrument -o compute_comm_overlap.inst -- compute_comm_overlap
 ${TOOL_COMMAND}-run -- ./compute_comm_overlap.inst 2
-cd ${TOOL_OUTPUT}-compute_comm_overlap.inst-output/
-ls *
-
+# Check for a real .proto artifact instead of matching tool output. The output
+# directory name differs between omnitrace and rocprofiler-systems, so glob on
+# the shared suffix and require a .proto inside it.
+assert_proto_output() {
+  local outdir n
+  outdir="$(ls -d ./*-compute_comm_overlap.inst-output 2>/dev/null | head -1)"
+  if [[ -z "${outdir}" || ! -d "${outdir}" ]]; then
+    echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: FAIL no *-compute_comm_overlap.inst-output directory produced"
+    return 1
+  fi
+  # The tool nests a timestamped subdirectory, so recurse.
+  n="$(find "${outdir}" -type f -name '*.proto' | wc -l)"
+  find "${outdir}" -type f | sed 's/^/    /' | head -20
+  if [[ "${n}" -gt 0 ]]; then
+    echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: PASS ${n} proto file(s) under ${outdir}"
+  else
+    echo "ROCPROFSYS_STREAM_OVERLAP_RESULT: FAIL 0 proto files under ${outdir}"
+    return 1
+  fi
+}
+assert_proto_output
 cd ..
 rm -rf ${BUILD_DIR}
 

tests/tau_exec_check.sh

@@ -84,10 +84,21 @@ export TAU_PROFILE=${TAU_PROFILE}
 export TAU_TRACE=${TAU_TRACE}
 
 WORKDIR=$(mktemp -d -p ${SRCDIR} build_XXXXXX)
+# Install the cleanup trap before the build gate below, which can exit early.
+trap 'cd "${SRCDIR}" && rm -rf "${WORKDIR}"' EXIT
 cp ${SRCDIR}/*.hip ${SRCDIR}/*.hpp ${SRCDIR}/*.h ${SRCDIR}/Makefile ${SRCDIR}/input.txt ${WORKDIR}/
 cd ${WORKDIR}
 
-make
+# Stop the test if the build fails, otherwise tau_exec would go on to profile a
+# binary that was never produced.
+if ! make; then
+   echo "TAU_CHECK: build FAILED, so nothing was profiled"
+   exit 1
+fi
+if [ ! -x ./Jacobi_hip ]; then
+   echo "TAU_CHECK: build reported success but ./Jacobi_hip is missing"
+   exit 1
+fi
 
 ROCM_VERSION=`cat ${ROCM_PATH}/.info/version | head -1 | cut -f1 -d'-' `
 
@@ -108,7 +119,7 @@ result=""; ver_gt "${ROCM_VERSION}" 6.1.9 && result="${ROCM_VERSION}"; echo $res
 # ("MPI processes (2) doesn't match the topology size (1)"). Only a bare Cray
 # MPICH (mpicc with no co-located launcher) falls back to srun. The MPICH hydra
 # mpiexec also rejects OpenMPI's --oversubscribe, so add that flag for OpenMPI
-# only (detected via ompi_info).
+# only (detected from the MPI wrapper below).
 MPI_BINDIR=$(dirname "$(command -v mpicc 2>/dev/null)" 2>/dev/null)
 if [ -n "${MPI_BINDIR}" ] && [ -x "${MPI_BINDIR}/mpirun" ]; then
    MPI_LAUNCH="${MPI_BINDIR}/mpirun -n 2"
@@ -117,10 +128,39 @@ elif [ -n "${MPI_BINDIR}" ] && [ -x "${MPI_BINDIR}/mpiexec" ]; then
 else
    MPI_LAUNCH="srun -n 2"
 fi
-if command -v ompi_info >/dev/null 2>&1; then
+# Ask the launch wrapper which MPI it is: only OpenMPI answers --showme:version
+# and only MPICH answers -compile_info. This is more reliable than checking PATH,
+# since a module can put one MPI's wrappers in front of another's.
+MPI_FAMILY=unknown
+if [ -n "${MPI_BINDIR}" ]; then
+   if "${MPI_BINDIR}/mpicc" --showme:version >/dev/null 2>&1; then
+      MPI_FAMILY=openmpi
+   elif "${MPI_BINDIR}/mpicc" -compile_info >/dev/null 2>&1; then
+      MPI_FAMILY=mpich
+   fi
+fi
+echo "TAU_CHECK: launching with ${MPI_FAMILY} wrappers from ${MPI_BINDIR:-<none, using srun>}"
+if [ "${MPI_FAMILY}" = "openmpi" ]; then
    MPI_LAUNCH="${MPI_LAUNCH} --oversubscribe"
 fi
 
+# Run both ranks on this node. The test needs two ranks on two GPUs, not two
+# nodes, and Jacobi is built in a directory inside the checkout. If that checkout
+# is on node-local storage, a multi-node launch could place a rank on a node that
+# cannot see the build directory. Hydra takes -hosts, OpenMPI takes --host with a
+# slot count. The srun fallback is left as is, because pinning it would also need
+# the step's GRES respecified.
+MPI_HOST_LOCAL="$(hostname -s)"
+case "${MPI_LAUNCH}" in
+   srun*) ;;
+   *) case "${MPI_FAMILY}" in
+         openmpi) MPI_LAUNCH="${MPI_LAUNCH} --host ${MPI_HOST_LOCAL}:2" ;;
+         mpich)   MPI_LAUNCH="${MPI_LAUNCH} -hosts ${MPI_HOST_LOCAL}" ;;
+         *) echo "TAU_CHECK: unknown MPI family, not pinning ranks to ${MPI_HOST_LOCAL}" ;;
+      esac ;;
+esac
+echo "TAU_CHECK: launcher is ${MPI_LAUNCH}"
+
 # Use a 1024x1024 local mesh so the TAU trace buffer fits in GPU memory
 # (default 4096x4096 caused "HIP failure: 'out of memory'" during trace finalization).
 if [[ "${result}" ]]; then
@@ -130,8 +170,42 @@ else
 fi
 
 ls
-pprof
+pprof 2>&1 | tee pprof.out
+
+# Check for the produced files and print our own result strings, which CMake
+# matches on. Matching the tool's own words was unreliable: "profile.0" also
+# appears in messages like "Could not open profile.0.0.0".
+tau_have() { ls $1 >/dev/null 2>&1; }
+tau_status=0
+
+if [ "${TAU_PROFILE}" = "1" ]; then
+   if tau_have 'profile.*'; then
+      echo "TAU_CHECK: profile artifact PRESENT"
+   else
+      echo "TAU_CHECK: profile artifact absent"
+      tau_status=1
+   fi
+   # Reported but not fatal on their own: each is a different test's subject, so
+   # the per-test expression decides rather than failing all of them together.
+   if grep -q 'MPI_Allreduce' pprof.out; then
+      echo "TAU_CHECK: MPI routines PRESENT in profile"
+   else
+      echo "TAU_CHECK: MPI routines absent from profile"
+   fi
+   if grep -q 'hipMemcpy' pprof.out; then
+      echo "TAU_CHECK: HIP routines PRESENT in profile"
+   else
+      echo "TAU_CHECK: HIP routines absent from profile"
+   fi
+fi
 
-cd ..
-rm -rf ${WORKDIR}
+if [ "${TAU_TRACE}" = "1" ]; then
+   if tau_have 'tautrace.*'; then
+      echo "TAU_CHECK: trace artifact PRESENT"
+   else
+      echo "TAU_CHECK: trace artifact absent"
+      tau_status=1
+   fi
+fi
 
+exit ${tau_status}

tests/tau_rccl_pc_check.sh

@@ -113,7 +113,20 @@ elif [ -n "${MPI_BINDIR}" ] && [ -x "${MPI_BINDIR}/mpiexec" ]; then
 else
    MPI_LAUNCH="srun -n 2"
 fi
-if command -v ompi_info >/dev/null 2>&1; then
+# Detect the MPI flavour from the launch wrapper; see tau_exec_check.sh. Checking
+# PATH instead could add OpenMPI's --oversubscribe to MPICH's mpiexec, which
+# rejects it. This does not fix the case where TAU and the prebuilt rccl-tests
+# binary come from different MPIs.
+MPI_FAMILY=unknown
+if [ -n "${MPI_BINDIR}" ]; then
+   if "${MPI_BINDIR}/mpicc" --showme:version >/dev/null 2>&1; then
+      MPI_FAMILY=openmpi
+   elif "${MPI_BINDIR}/mpicc" -compile_info >/dev/null 2>&1; then
+      MPI_FAMILY=mpich
+   fi
+fi
+echo "MPI family of ${MPI_BINDIR:-<none, using srun>}: ${MPI_FAMILY}"
+if [ "${MPI_FAMILY}" = "openmpi" ]; then
    MPI_LAUNCH="${MPI_LAUNCH} --oversubscribe"
 fi