From 6580ee96b018cd5066c0f9e449f45ac07a830132 Mon Sep 17 00:00:00 2001 From: Agustin Palazuelos Date: Tue, 28 Apr 2026 16:46:20 -0300 Subject: [PATCH 1/4] Add pt-mongodb-stalk tool and README --- bin/pt-mongodb-stalk | 1656 +++++++++++++++++++++++++++++++++++ bin/pt-mongodb-stalk.README | 172 ++++ 2 files changed, 1828 insertions(+) create mode 100755 bin/pt-mongodb-stalk create mode 100644 bin/pt-mongodb-stalk.README diff --git a/bin/pt-mongodb-stalk b/bin/pt-mongodb-stalk new file mode 100755 index 000000000..216f5500c --- /dev/null +++ b/bin/pt-mongodb-stalk @@ -0,0 +1,1656 @@ +#!/usr/bin/env bash + +# This program is part of Percona Toolkit: http://www.percona.com/software/ +# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal +# notices and disclaimers. + +set -u + +# ########################################################################### +# log_warn_die package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the GitHub repository at, +# lib/bash/log_warn_die.sh +# t/lib/bash/log_warn_die.sh +# See https://github.com/percona/percona-toolkit for more information. +# ########################################################################### + +PTFUNCNAME="" +PTDEBUG="${PTDEBUG:-""}" +EXIT_STATUS=0 + +ts() { + TS=$(date +%F-%T | tr ':-' '_') + echo "$TS $*" +} + +info() { + [ "${OPT_VERBOSE:-3}" -ge 3 ] && ts "$*" +} + +log() { + [ "${OPT_VERBOSE:-3}" -ge 2 ] && ts "$*" +} + +warn() { + [ "${OPT_VERBOSE:-3}" -ge 1 ] && ts "$*" >&2 + EXIT_STATUS=1 +} + +die() { + ts "$*" >&2 + EXIT_STATUS=1 + exit 1 +} + +_d () { + [ "$PTDEBUG" ] && echo "# $PTFUNCNAME: $(ts "$*")" >&2 +} + +# ########################################################################### +# End log_warn_die package +# ########################################################################### + +# ########################################################################### +# subshell package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the GitHub repository at, +# lib/bash/subshell.sh +# t/lib/bash/subshell.sh +# See https://github.com/percona/percona-toolkit for more information. +# ########################################################################### + + +wait_for_subshells() { + local max_wait=$1 + if [ "$(jobs)" ]; then + log "Waiting up to $max_wait seconds for subprocesses to finish..." + local slept=0 + while [ -n "$(jobs)" ]; do + local subprocess_still_running="" + for pid in $(jobs -p); do + if kill -0 "$pid" >/dev/null 2>&1; then + subprocess_still_running=1 + fi + done + if [ "$subprocess_still_running" ]; then + sleep 1 + slept=$((slept + 1)) + [ "$slept" -ge "$max_wait" ] && break + else + break + fi + done + fi +} + +kill_all_subshells() { + if [ "$(jobs)" ]; then + for pid in $(jobs -p); do + if kill -0 "$pid" >/dev/null 2>&1; then + log "Killing subprocess $pid" + kill "$pid" >/dev/null 2>&1 + fi + done + fi +} + +# ########################################################################### +# End subshell package +# ########################################################################### + +# ########################################################################### +# parse_options package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the GitHub repository at, +# lib/bash/parse_options.sh +# t/lib/bash/parse_options.sh +# See https://github.com/percona/percona-toolkit for more information. +# ########################################################################### + +ARGV="" # Non-option args (probably input files) +EXT_ARGV="" # Everything after -- (args for an external command) +HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV +OPT_ERRS=0 # How many command line option errors +OPT_VERSION="" # If --version was specified +OPT_HELP="" # If --help was specified +OPT_ASK_PASS="" # If --ask-pass was specified +PO_DIR="" # Directory with program option spec files +GLOBAL_CONFIG=0 # We ignore non-recognized options in global configs + +usage() { + local file="$1" + + local usage="$(grep '^Usage: ' "$file")" + echo "$usage" + echo + echo "For more information, 'man $TOOL' or 'perldoc $file'." +} + +usage_or_errors() { + local file="$1" + + if [ "$OPT_VERSION" ]; then + local version + version=$(grep '^pt-[^ ]\+ [0-9]' "$file") + echo "$version" + return 1 + fi + + if [ -z "$(_which perl)" ]; then + echo "Perl binary required to run this tool" + return 1 + fi + + if [ "$OPT_HELP" ]; then + usage "$file" + echo + echo "Command line options:" + echo + perl -e ' + use strict; + use warnings FATAL => qw(all); + my $lcol = 24; + my $rcol = 80 - $lcol; + my $name; + while ( <> ) { + my $line = $_; + chomp $line; + if ( $line =~ s/^long:/ --/ ) { + $name = $line; + } + elsif ( $line =~ s/^desc:// ) { + $line =~ s/ +$//mg; + my @lines = grep { $_ } + $line =~ m/(.{0,$rcol})(?:\s+|\Z)/g; + if ( length($name) >= $lcol ) { + print $name, "\n", (q{ } x $lcol); + } + else { + printf "%-${lcol}s", $name; + } + print join("\n" . (q{ } x $lcol), @lines); + print "\n"; + } + } + ' "$PO_DIR"/* + echo + echo "Examples:" + echo + perl -e ' + use strict; + use warnings FATAL => qw(all); + my $in = 0; + while ( <> ) { + if ( /^=head1 EXAMPLES/ ) { + $in = 1; + next; + } + last if $in && /^=head1 /; + next unless $in; + next if /^=cut/; + next if /^=over/; + next if /^=back/; + next if /^=pod/; + s/^C<([^>]+)>/$1/g; + s/L<"[^"]+">//g; + print; + } + ' "$file" + return 1 + fi + + if [ "$OPT_ERRS" -gt 0 ]; then + echo + usage "$file" + return 1 + fi + + return 0 +} + +option_error() { + local err="$1" + OPT_ERRS=$((OPT_ERRS + 1)) + echo "$err" >&2 +} + +parse_options() { + local file="$1" + shift + + ARGV="" + EXT_ARGV="" + HAVE_EXT_ARGV="" + OPT_ERRS=0 + OPT_VERSION="" + OPT_HELP="" + OPT_ASK_PASS="" + PO_DIR="$PT_TMPDIR/po" + + [ -d "$PO_DIR" ] || mkdir "$PO_DIR" || die "Cannot mkdir $PO_DIR" + rm -rf "$PO_DIR"/* || die "Cannot rm -rf $PO_DIR/*" + + _parse_pod "$file" + _eval_po + + if [ $# -ge 2 ] && [ "$1" = "--config" ]; then + shift + local user_config_files="$1" + shift + local IFS="," + for user_config_file in $user_config_files; do + _parse_config_files "$user_config_file" + done + else + GLOBAL_CONFIG=1 + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" + GLOBAL_CONFIG=0 + _parse_config_files "/etc/percona-toolkit/$TOOL.conf" + if [ "${HOME:-}" ]; then + GLOBAL_CONFIG=1 + _parse_config_files "$HOME/.percona-toolkit.conf" + GLOBAL_CONFIG=0 + _parse_config_files "$HOME/.$TOOL.conf" + fi + fi + + _parse_command_line "${@:-""}" +} + +_parse_pod() { + local file="$1" + + PO_FILE="$file" PO_DIR="$PO_DIR" perl -e ' + $/ = ""; + my $file = $ENV{PO_FILE}; + open my $fh, "<", $file or die "Cannot open $file: $!"; + while ( defined(my $para = <$fh>) ) { + next unless $para =~ m/^=head1 OPTIONS/; + while ( defined(my $para = <$fh>) ) { + last if $para =~ m/^=head1/; + chomp; + if ( $para =~ m/^=item --(\S+)/ ) { + my $opt = $1; + my $file = "$ENV{PO_DIR}/$opt"; + open my $opt_fh, ">", $file or die "Cannot open $file: $!"; + print $opt_fh "long:$opt\n"; + $para = <$fh>; + chomp; + if ( $para =~ m/^[a-z ]+:/ ) { + map { + chomp; + my ($attrib, $val) = split(/: /, $_); + print $opt_fh "$attrib:$val\n"; + } split(/; /, $para); + $para = <$fh>; + chomp; + } + my ($desc) = $para =~ m/^([^?.]+)/; + print $opt_fh "desc:$desc.\n"; + close $opt_fh; + } + } + last; + } + ' +} + +_eval_po() { + local IFS=":" + for opt_spec in "$PO_DIR"/*; do + local opt="" + local default_val="" + local neg=0 + local size=0 + while read -r key val; do + case "$key" in + long) + opt=$(echo "$val" | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]') + ;; + default) + default_val="$val" + ;; + "short form"|desc) + ;; + type) + [ "$val" = "size" ] && size=1 + ;; + negatable) + [ "$val" = "yes" ] && neg=1 + ;; + *) + ;; + esac + done < "$opt_spec" + + [ -n "$opt" ] || die "No long attribute in option spec $opt_spec" + + if [ "$size" -eq 1 ] && [ -n "$default_val" ]; then + default_val=$(size_to_bytes "$default_val") + fi + + eval "OPT_${opt}"='"$default_val"' + if [ "$neg" -eq 1 ] && [ "$default_val" = "yes" ]; then + : + fi + done +} + +_parse_config_files() { + for config_file in "${@:-""}"; do + test -f "$config_file" || continue + while read -r config_opt; do + echo "$config_opt" | grep '^[ ]*[^#]' >/dev/null 2>&1 || continue + config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/\s[ ]*#.*$//')" + [ "$config_opt" = "" ] && continue + if ! [ "$HAVE_EXT_ARGV" ]; then + config_opt="--$config_opt" + fi + _parse_command_line "$config_opt" + done < "$config_file" + HAVE_EXT_ARGV="" + done +} + +_parse_command_line() { + local opt="" + local val="" + local next_opt_is_val="" + local opt_is_ok="" + local opt_is_negated="" + local real_opt="" + local required_arg="" + local spec="" + + for opt in "${@:-""}"; do + if [ "$opt" = "--" -o "$opt" = "----" ]; then + HAVE_EXT_ARGV=1 + continue + fi + if [ "$HAVE_EXT_ARGV" ]; then + if [ "$EXT_ARGV" ]; then + EXT_ARGV="$EXT_ARGV $opt" + else + EXT_ARGV="$opt" + fi + continue + fi + + if [ "$next_opt_is_val" ]; then + next_opt_is_val="" + if [ $# -eq 0 ] || [ "$(expr "$opt" : "\-")" -eq 1 ]; then + option_error "$real_opt requires a $required_arg argument" + continue + fi + val="$opt" + opt_is_ok=1 + else + if [ "$(expr "$opt" : "\-")" -eq 0 ]; then + ARGV="${ARGV:+$ARGV }$opt" + continue + fi + + real_opt="$opt" + if $(echo "$opt" | grep '^--no[^-]' >/dev/null); then + local base_opt + base_opt=$(echo "$opt" | sed 's/^--no//') + if [ -f "$PT_TMPDIR/po/$base_opt" ]; then + opt_is_negated=1 + opt="$base_opt" + else + opt=$(echo "$opt" | sed 's/^-*//') + fi + else + if $(echo "$opt" | grep '^--no-' >/dev/null); then + opt_is_negated=1 + opt=$(echo "$opt" | sed 's/^--no-//') + else + opt=$(echo "$opt" | sed 's/^-*//') + fi + fi + + if $(echo "$opt" | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then + val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')" + opt="$(echo "$opt" | awk -F= '{print $1}')" + fi + + if [ -f "$PT_TMPDIR/po/$opt" ]; then + spec="$PT_TMPDIR/po/$opt" + else + spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* 2>/dev/null | cut -d ':' -f 1) + if [ -z "$spec" ]; then + if [ "$GLOBAL_CONFIG" -eq 1 ]; then + continue + else + option_error "Unknown option: $real_opt" + continue + fi + fi + fi + + required_arg=$(awk -F: '/^type:/{print $2}' "$spec") + if [ "$required_arg" ]; then + if [ "$val" ]; then + opt_is_ok=1 + else + next_opt_is_val=1 + fi + else + if [ "$val" ]; then + option_error "Option $real_opt does not take a value" + continue + fi + if [ "$opt_is_negated" ]; then + val="" + else + val="yes" + fi + opt_is_ok=1 + fi + fi + + if [ "$opt_is_ok" ]; then + opt=$(grep '^long:' "$spec" | cut -d':' -f2 | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]') + if grep "^type:size" "$spec" >/dev/null; then + val=$(size_to_bytes "$val") + fi + eval "OPT_$opt"='$val' + opt="" + val="" + next_opt_is_val="" + opt_is_ok="" + opt_is_negated="" + real_opt="" + required_arg="" + spec="" + fi + done +} + +size_to_bytes() { + local size="$1" + echo "$size" | perl -ne '%f=(B=>1, K=>1_024, M=>1_048_576, G=>1_073_741_824, T=>1_099_511_627_776); m/^(\d+)([kMGT])?/i; print $1 * $f{uc($2 || "B")};' +} + +# ########################################################################### +# tmpdir package +# ########################################################################### + +PT_TMPDIR="" + +mk_tmpdir() { + local dir="${1:-""}" + if [ -n "$dir" ]; then + [ -d "$dir" ] || mkdir "$dir" || die "Cannot make tmpdir $dir" + PT_TMPDIR="$dir" + else + local tool="${0##*/}" + local pid="$$" + PT_TMPDIR=$(mktemp -d -t "${tool}.${pid}.XXXXXX") || die "Cannot make secure tmpdir" + fi +} + +rm_tmpdir() { + if [ -n "$PT_TMPDIR" ] && [ -d "$PT_TMPDIR" ]; then + rm -rf "$PT_TMPDIR" + fi + PT_TMPDIR="" +} + +remove_empty_err_files() { + local dir="$1" + [ -d "$dir" ] || return 0 + find "$dir" -maxdepth 1 -type f -name '*.err' -size 0 -delete 2>/dev/null || true +} + +# ########################################################################### +# alt_cmds package +# ########################################################################### + +_which() { + if [ -x /usr/bin/which ]; then + /usr/bin/which "$1" 2>/dev/null | awk '{print $1}' + elif which which 1>/dev/null 2>&1; then + which "$1" 2>/dev/null | awk '{print $1}' + else + echo "$1" + fi +} + +_pidof() { + local cmd="$1" + if ! pidof "$cmd" 2>/dev/null; then + ps -eo pid,ucomm | awk -v comm="$cmd" '$2 == comm { print $1 }' + fi +} + +# ########################################################################### +# safeguards package +# ########################################################################### + +disk_space() { + local filesystem="${1:-$PWD}" + df -P -k "$filesystem" +} + +check_disk_space() { + local file="$1" + local min_free_bytes="${2:-0}" + local min_free_pct="${3:-0}" + local bytes_margin="${4:-0}" + + local used_bytes free_bytes pct_used pct_free real_free_bytes real_pct_free + used_bytes=$(tail -n 1 "$file" | perl -ane 'print $F[2] * 1024') + free_bytes=$(tail -n 1 "$file" | perl -ane 'print $F[3] * 1024') + pct_used=$(tail -n 1 "$file" | perl -ane 'print ($F[4] =~ m/(\d+)/)') + pct_free=$((100 - pct_used)) + real_free_bytes=$free_bytes + real_pct_free=$pct_free + + if [ "$bytes_margin" -gt 0 ]; then + used_bytes=$((used_bytes + bytes_margin)) + free_bytes=$((free_bytes - bytes_margin)) + pct_used=$(perl -e "print int(($used_bytes/($used_bytes + $free_bytes)) * 100)") + pct_free=$((100 - pct_used)) + fi + + if [ "$free_bytes" -lt "$min_free_bytes" ] || [ "$pct_free" -lt "$min_free_pct" ]; then + warn "Not enough free disk space: + Limit: ${min_free_pct}% free, ${min_free_bytes} bytes free + Actual: ${real_pct_free}% free, ${real_free_bytes} bytes free (- $bytes_margin bytes margin) +" + cat "$file" >&2 + return 1 + fi + + return 0 +} + +# ########################################################################### +# daemon package +# ########################################################################### + +make_pid_file() { + local file="$1" + local pid="$2" + + if [ -f "$file" ]; then + local old_pid + old_pid=$(cat "$file") + if [ -z "$old_pid" ]; then + die "PID file $file already exists but it is empty" + else + kill -0 "$old_pid" 2>/dev/null + if [ $? -eq 0 ]; then + die "PID file $file already exists and its PID ($old_pid) is running" + fi + fi + fi + + echo "$pid" > "$file" || die "Cannot create or write PID file $file" +} + +remove_pid_file() { + local file="$1" + [ -f "$file" ] && rm "$file" +} + +# ########################################################################### +# Mongo helpers +# ########################################################################### + +CMD_MONGOSH="${CMD_MONGOSH:-"$(_which mongosh)"}" +CMD_MONGO="${CMD_MONGO:-"$(_which mongo)"}" +CMD_MONGOSTAT="${CMD_MONGOSTAT:-"$(_which mongostat)"}" +CMD_MONGOTOP="${CMD_MONGOTOP:-"$(_which mongotop)"}" +CMD_IOSTAT="${CMD_IOSTAT:-"$(_which iostat)"}" +CMD_VMSTAT="${CMD_VMSTAT:-"$(_which vmstat)"}" +CMD_MPSTAT="${CMD_MPSTAT:-"$(_which mpstat)"}" +CMD_PIDSTAT="${CMD_PIDSTAT:-"$(_which pidstat)"}" +CMD_TOP="${CMD_TOP:-"$(_which top)"}" +CMD_PMAP="${CMD_PMAP:-"$(_which pmap)"}" +CMD_DMESG="${CMD_DMESG:-"$(_which dmesg)"}" +CMD_JOURNALCTL="${CMD_JOURNALCTL:-"$(_which journalctl)"}" + +MONGO_SHELL="" +MONGO_EVAL_PREFIX="" +MONGO_AUTH_DB="" +MONGO_CONNECT_ARGS="" +MONGO_MONITOR_ARGS="" +MONGO_REPLSET_NAME="" +MONGO_REPLICA_SET_ROLE="" +MONGO_PROCESS_TYPE="" +MONGO_CLUSTER_ROLE="" + +choose_mongo_shell() { + if [ -n "$CMD_MONGOSH" ]; then + MONGO_SHELL="$CMD_MONGOSH" + MONGO_EVAL_PREFIX="EJSON.stringify" + elif [ -n "$CMD_MONGO" ]; then + MONGO_SHELL="$CMD_MONGO" + MONGO_EVAL_PREFIX="JSON.stringify" + else + die "Cannot execute mongosh or mongo. Check that one is in PATH." + fi +} + +mongo_connect_args() { + local args="" + if [ -n "$OPT_URI" ]; then + args="$OPT_URI" + else + args="mongodb://${OPT_HOST}:${OPT_PORT}/admin" + fi + + if [ -n "$OPT_USER" ]; then + args="$args --username=$OPT_USER" + fi + if [ -n "$OPT_PASSWORD" ]; then + args="$args --password=$OPT_PASSWORD" + fi + if [ -n "$OPT_AUTHENTICATIONDATABASE" ]; then + args="$args --authenticationDatabase=$OPT_AUTHENTICATIONDATABASE" + fi + if [ -n "$OPT_SSLCAFILE" ]; then + args="$args --tlsCAFile=$OPT_SSLCAFILE" + fi + if [ -n "$OPT_SSLPEMKEYFILE" ]; then + args="$args --tlsCertificateKeyFile=$OPT_SSLPEMKEYFILE" + fi + if [ "$OPT_TLS" ]; then + args="$args --tls" + fi + + echo "$args" +} + +mongo_eval() { + local script="$1" + "$MONGO_SHELL" --quiet $MONGO_CONNECT_ARGS --eval "$script" +} + +mongo_monitor_args() { + local args="" + + if [ -n "$OPT_USER" ]; then + args="$args --username=$OPT_USER" + fi + if [ -n "$OPT_PASSWORD" ]; then + args="$args --password=$OPT_PASSWORD" + fi + if [ -n "$OPT_AUTHENTICATIONDATABASE" ]; then + args="$args --authenticationDatabase=$OPT_AUTHENTICATIONDATABASE" + fi + if [ -n "$OPT_SSLCAFILE" ]; then + args="$args --sslCAFile=$OPT_SSLCAFILE" + fi + if [ -n "$OPT_SSLPEMKEYFILE" ]; then + args="$args --sslPEMKeyFile=$OPT_SSLPEMKEYFILE" + fi + if [ "$OPT_TLS" ]; then + args="$args --ssl" + fi + if [ -n "$OPT_URI" ]; then + args="$args $OPT_URI" + else + args="$args --host=$OPT_HOST --port=$OPT_PORT" + fi + + echo "$args" +} + +mongo_monitor_exec() { + local cmd="$1" + local interval="$2" + shift 2 + "$cmd" "$@" $MONGO_MONITOR_ARGS "$interval" +} + +mongo_admin_json() { + local expr="$1" + mongo_eval " + const __pt = db.getSiblingDB('admin'); + const __doc = ${expr}; + if (__doc && typeof __doc === 'object' && !Array.isArray(__doc)) { + delete __doc.ok; + delete __doc['\$clusterTime']; + delete __doc.operationTime; + } + print(${MONGO_EVAL_PREFIX}(__doc)); + " +} + +detect_mongo_topology() { + local topology + topology=$(mongo_eval " + const __pt = db.getSiblingDB('admin'); + const __serverStatus = __pt.runCommand({serverStatus: 1}) || {}; + let __clusterRole = ''; + let __hello = {}; + try { + const __cmdLineOpts = __pt.runCommand({getCmdLineOpts: 1}); + if (__cmdLineOpts && __cmdLineOpts.parsed && __cmdLineOpts.parsed.sharding && __cmdLineOpts.parsed.sharding.clusterRole) { + __clusterRole = __cmdLineOpts.parsed.sharding.clusterRole; + } + } catch (e) { + } + try { + __hello = __pt.runCommand({hello: 1}) || {}; + } catch (e) { + try { + __hello = __pt.runCommand({isMaster: 1}) || {}; + } catch (e2) { + __hello = {}; + } + } + + let __replicaSetRole = ''; + if (__hello.setName) { + if (__hello.isWritablePrimary || __hello.ismaster) { + __replicaSetRole = 'PRIMARY'; + } else if (__hello.secondary) { + __replicaSetRole = 'SECONDARY'; + } else if (__hello.arbiterOnly) { + __replicaSetRole = 'ARBITER'; + } else { + __replicaSetRole = 'MEMBER'; + } + } + + if (__clusterRole === 'shardsvr') { + __clusterRole = 'shard'; + } else if (__clusterRole === 'configsvr') { + __clusterRole = 'config-server'; + } + + print([ + __serverStatus.process || '', + __clusterRole, + __replicaSetRole, + __hello.setName || '' + ].join('\t')); + ") + + IFS=$'\t' read -r MONGO_PROCESS_TYPE MONGO_CLUSTER_ROLE MONGO_REPLICA_SET_ROLE MONGO_REPLSET_NAME </dev/null | awk '/LISTEN/ && ($1=="mongod" || $1=="mongos") {print $2; exit}') + fi + if [ -z "$pid" ]; then + pid=$(_pidof mongod | awk '{print $1; exit}') + fi + if [ -z "$pid" ]; then + pid=$(_pidof mongos | awk '{print $1; exit}') + fi + echo "$pid" +} + +run_maybe_sudo() { + local cmd="$1" + shift + + if [ "$(id -u)" -eq 0 ]; then + "$cmd" "$@" + elif [ -n "$(_which sudo)" ] && sudo -n true >/dev/null 2>&1; then + sudo -n "$cmd" "$@" + else + "$cmd" "$@" + fi +} + +sample_ts() { + date +%F-%T | tr ':-' '_' +} + +sample_file() { + local d="$1" + local label="$2" + local ext="$3" + printf '%s/%s-%s.%s' "$d" "$(sample_ts)" "$label" "$ext" +} + +sample_file_with_ts() { + local d="$1" + local ts="$2" + local label="$3" + local ext="$4" + printf '%s/%s-%s.%s' "$d" "$ts" "$label" "$ext" +} + +state_file() { + local d="$1" + local label="$2" + printf '%s/%s' "$d" "$label" +} + +fixed_file() { + local d="$1" + local label="$2" + local ext="$3" + printf '%s/%s.%s' "$d" "$label" "$ext" +} + +sample_count() { + perl -e ' + my ($run_time, $sleep_collect) = @ARGV; + $run_time = 0 + ($run_time || 0); + $sleep_collect = 0 + ($sleep_collect || 0); + $sleep_collect = 1 if $sleep_collect <= 0; + my $count = int(($run_time / $sleep_collect) + 0.999999); + $count = 1 if $count < 1; + print $count; + ' "$1" "$2" +} + +collect_mongo_monitor_sample() { + local d="$1" + local label="$2" + local cmd="$3" + local interval="$4" + shift 4 + + local ts out err status + ts="$(sample_ts)" + out="$(sample_file_with_ts "$d" "$ts" "$label" "txt")" + err="$(sample_file_with_ts "$d" "$ts" "$label" "err")" + + mongo_monitor_exec "$cmd" "$interval" "$@" > "$out" 2>"$err" + status=$? + + if [ "$status" -ne 0 ]; then + echo "Command exited with status $status: $cmd $* $MONGO_MONITOR_ARGS $interval" >> "$err" + fi + if [ ! -s "$out" ]; then + echo "Command produced no output: $cmd $* $MONGO_MONITOR_ARGS $interval" >> "$err" + fi +} + +collect_command_sample() { + local d="$1" + local label="$2" + local cmd="$3" + shift 3 + + local ts out err status + ts="$(sample_ts)" + out="$(sample_file_with_ts "$d" "$ts" "$label" "txt")" + err="$(sample_file_with_ts "$d" "$ts" "$label" "err")" + + "$cmd" "$@" > "$out" 2>"$err" + status=$? + + if [ "$status" -ne 0 ]; then + echo "Command exited with status $status: $cmd $*" >> "$err" + fi + if [ ! -s "$out" ]; then + echo "Command produced no output: $cmd $*" >> "$err" + fi +} + +# ########################################################################### +# collection package +# ########################################################################### + +collect() { + local d="$1" + local cnt + local collect_wait + local mongo_pid="" + + cnt=$(sample_count "$OPT_RUN_TIME" "$OPT_SLEEP_COLLECT") + collect_wait="$OPT_RUN_TIME" + [ "$collect_wait" -lt 1 ] && collect_wait=1 + mongo_pid=$(get_mongo_pid) + collect_mongodb_data_one "$d" "$mongo_pid" + collect_system_data "$d" "$mongo_pid" "$cnt" + + log "Loop start: $(date +'TS %s.%N %F %T')" + collect_system_data_loop "$d" "$mongo_pid" + collect_mongodb_data_loop "$d" + date +"TS %s.%N %F %T" >> "$(state_file "$d" "heartbeat")" + + collect_mongodb_data_two "$d" + + wait_for_subshells "$collect_wait" + kill_all_subshells +} + +collect_mongodb_data_one() { + local d="$1" + local mongo_pid="$2" + + detect_mongo_topology + + : +} + +collect_mongodb_data_loop() { + local d="$1" + (mongo_admin_json "db.runCommand({serverStatus: 1})" > "$(sample_file "$d" "serverStatus" "json")") & + (mongo_admin_json "db.runCommand({currentOp: 1, active: true, idleConnections: false})" > "$(sample_file "$d" "currentOp" "json")") & +} + +collect_mongodb_data_two() { + : +} + +collect_system_data() { + local d="$1" + local mongo_pid="$2" + local cnt="$3" + + if [ -f /var/log/messages ]; then + (run_maybe_sudo cp /var/log/messages "$(fixed_file "$d" "messages" "out")") 2>"$(fixed_file "$d" "messages" "err")" & + fi + + if [ -n "$CMD_VMSTAT" ]; then + (collect_command_sample "$d" "vmstat" "$CMD_VMSTAT" "$OPT_SLEEP_COLLECT" "$cnt") & + fi + if [ -n "$CMD_IOSTAT" ]; then + (collect_command_sample "$d" "iostat" "$CMD_IOSTAT" "$OPT_SLEEP_COLLECT" "$cnt") & + fi + if [ -n "$CMD_MPSTAT" ]; then + (collect_command_sample "$d" "mpstat" "$CMD_MPSTAT" -P ALL "$OPT_SLEEP_COLLECT" "$cnt") & + fi + if [ -n "$CMD_TOP" ]; then + if top -bn1 >/dev/null 2>&1; then + (collect_command_sample "$d" "top" "$CMD_TOP" -b -d "$OPT_SLEEP_COLLECT" -n "$cnt") & + elif top -l 1 >/dev/null 2>&1; then + (collect_command_sample "$d" "top" "$CMD_TOP" -l "$cnt" -s "$OPT_SLEEP_COLLECT") & + fi + fi + if [ -n "$CMD_MONGOSTAT" ]; then + (collect_mongo_monitor_sample "$d" "mongostat" "$CMD_MONGOSTAT" "$OPT_SLEEP_COLLECT" --rowcount "$cnt") & + fi + if [ -n "$CMD_MONGOTOP" ]; then + (collect_mongo_monitor_sample "$d" "mongotop" "$CMD_MONGOTOP" "$OPT_SLEEP_COLLECT" --rowcount "$cnt") & + fi +} + +collect_system_data_loop() { + local d="$1" + local mongo_pid="$2" + local disk_space_file="$PT_TMPDIR/collect-disk-space" + local mongo_proc_label="pidstat_mongod" + + disk_space "$d" > "$disk_space_file" + check_disk_space "$disk_space_file" "$OPT_DISK_BYTES_FREE" "$OPT_DISK_PCT_FREE" || return + + (ps faux > "$(sample_file "$d" "ps" "txt")") & + if [ -n "$CMD_PIDSTAT" ]; then + (collect_command_sample "$d" "pidstat_d" "$CMD_PIDSTAT" -d 1 1) & + (collect_command_sample "$d" "pidstat_u" "$CMD_PIDSTAT" -u 1 1) & + if [ -n "$mongo_pid" ]; then + [ "$MONGO_PROCESS_TYPE" = "mongos" ] && mongo_proc_label="pidstat_mongos" + (collect_command_sample "$d" "$mongo_proc_label" "$CMD_PIDSTAT" -urdwt -h -p "$mongo_pid" 1 1) & + fi + fi +} + +# ########################################################################### +# Global variables +# ########################################################################### + +TRIGGER_FUNCTION="" +RAN_WITH="" +EXIT_REASON="" +TOOL="pt-mongodb-stalk" +OKTORUN=1 +ITER=1 + +# ########################################################################### +# Plugin hooks +# ########################################################################### + +before_stalk() { :; } +before_collect() { :; } +after_collect() { :; } +after_collect_sleep() { :; } +after_interval_sleep() { :; } +after_stalk() { :; } + +# ########################################################################### +# Trigger and control flow +# ########################################################################### + +set_trg_func() { + local func="$1" + if [ -f "$func" ]; then + . "$func" + TRIGGER_FUNCTION="trg_plugin" + return 0 + fi + + func=$(echo "$func" | tr '[:upper:]' '[:lower:]') + if [ "$func" = "status" ] || [ "$func" = "currentop" ]; then + TRIGGER_FUNCTION="trg_$func" + return 0 + fi + + return 1 +} + +trg_status() { + local var="$1" + mongo_status_value "$var" +} + +trg_currentop() { + local var="$1" + mongo_currentop_count "$var" +} + +oktorun() { + if [ "$OKTORUN" -eq 0 ]; then + [ -z "$EXIT_REASON" ] && EXIT_REASON="OKTORUN is false" + return 1 + fi + + if [ -n "${OPT_ITERATIONS:-}" ] && [ "$OPT_ITERATIONS" -gt 0 ] && [ "$ITER" -gt "$OPT_ITERATIONS" ]; then + [ -z "$EXIT_REASON" ] && EXIT_REASON="no more iterations" + return 1 + fi + + return 0 +} + +sleep_ok() { + local seconds="$1" + local msg="${2:-""}" + if oktorun; then + [ "$msg" ] && log "$msg" + sleep "$seconds" + fi +} + +purge_samples() { + local dir + dir=$(realpath -s "$1") + local retention_time="$2" + local retention_count="$3" + + find "$dir" -maxdepth 1 -type f -mtime +"$retention_time" -regextype posix-egrep \ + -regex "$dir/[0-9]{4}(_[0-9]{2}){5}-.*" -exec rm -f '{}' \; + + if [ "$retention_count" -gt 0 ]; then + local target_count=$((retention_count + 1)) + local files_to_delete + files_to_delete=$(find "$dir" -maxdepth 1 -type f -exec basename {} \; | cut -f1 -d- | sort -r | uniq | tail -n +"${target_count}") + for prefix in $files_to_delete; do + rm -f "$dir"/"$prefix"-* 2>/dev/null + done + fi +} + +sigtrap() { + if [ "$OKTORUN" -eq 1 ]; then + warn "Caught signal, exiting" + OKTORUN=0 + else + warn "Caught signal again, forcing exit" + exit "$EXIT_STATUS" + fi +} + +stalk() { + local cycles_true=0 + local matched="" + local last_prefix="" + + while oktorun; do + local msg="" + if [ "$OPT_STALK" ]; then + local value + value=$($TRIGGER_FUNCTION "$OPT_VARIABLE") + local trg_exit_status=$? + if [ -z "$value" ]; then + warn "Detected value is empty; something failed? Trigger exit status: $trg_exit_status" + matched="" + cycles_true=0 + elif (( $(echo "$value $OPT_THRESHOLD" | awk '{print ($1 > $2)}') )); then + matched="yes" + cycles_true=$((cycles_true + 1)) + else + matched="" + cycles_true=0 + fi + + msg="Check results: $OPT_FUNCTION($OPT_VARIABLE)=$value, matched=${matched:-no}, cycles_true=$cycles_true" + if [ "$matched" ]; then + log "$msg" + else + info "$msg" + fi + elif [ "$OPT_COLLECT" ]; then + matched=1 + cycles_true=$OPT_CYCLES + msg="Not stalking; collect triggered immediately" + log "$msg" + fi + + if [ "$matched" ] && [ "$cycles_true" -ge "$OPT_CYCLES" ]; then + log "Collect $ITER triggered" + + if [ "$OPT_COLLECT" ]; then + local prefix="${OPT_PREFIX:-$(date +%F-%T | tr ':-' '_')}" + local margin="20971520" + if [ -n "$last_prefix" ]; then + margin=$(du -mc "$OPT_DEST"/"$last_prefix"-* 2>/dev/null | tail -n 1 | awk '{print $1}') + [ -n "$margin" ] || margin="20971520" + fi + + disk_space "$OPT_DEST" > "$PT_TMPDIR/$prefix-disk-space" + check_disk_space "$PT_TMPDIR/$prefix-disk-space" "$OPT_DISK_BYTES_FREE" "$OPT_DISK_PCT_FREE" "$margin" + if [ $? -eq 0 ]; then + ts "$msg" >> "$(state_file "$OPT_DEST" "trigger")" + ts "pt-mongodb-stalk ran with $RAN_WITH" >> "$(state_file "$OPT_DEST" "trigger")" + last_prefix="$prefix" + + before_collect + + local collect_start_epoch + local collect_end_epoch + local collect_elapsed + collect_start_epoch=$(date +%s) + ts "Collect $ITER started; run-time=$OPT_RUN_TIME; sleep-collect=$OPT_SLEEP_COLLECT" >> "$(state_file "$OPT_DEST" "trigger")" + + ( + collect "$OPT_DEST" + ) >> "$(state_file "$OPT_DEST" "log")" 2>&1 & + local collector_pid=$! + log "Collect $ITER PID $collector_pid" + after_collect "$collector_pid" + wait "$collector_pid" + local collector_status=$? + if [ "$collector_status" -ne 0 ]; then + warn "Collect $ITER exited with status $collector_status" + fi + collect_end_epoch=$(date +%s) + collect_elapsed=$((collect_end_epoch - collect_start_epoch)) + ts "Collect $ITER finished in ${collect_elapsed}s; sleep=$OPT_SLEEP before next iteration" >> "$(state_file "$OPT_DEST" "trigger")" + else + warn "Collect canceled because there will not be enough disk space after collecting another sample" + fi + + if [ -d "$OPT_DEST" ]; then + purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" + fi + fi + + log "Collect $ITER done" + ITER=$((ITER + 1)) + cycles_true=0 + sleep_ok "$OPT_SLEEP" "Sleeping $OPT_SLEEP seconds after collect" + after_collect_sleep + else + sleep_ok "$OPT_INTERVAL" + after_interval_sleep + fi + done + + if [ "$OPT_COLLECT" ] && [ -d "$OPT_DEST" ]; then + purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" + fi + + wait_for_subshells $((OPT_RUN_TIME * 3)) + kill_all_subshells + remove_empty_err_files "$OPT_DEST" +} + +main() { + trap sigtrap SIGHUP SIGINT SIGTERM + + RAN_WITH="--function=$OPT_FUNCTION --variable=$OPT_VARIABLE --threshold=$OPT_THRESHOLD --match=$OPT_MATCH --cycles=$OPT_CYCLES --interval=$OPT_INTERVAL --iterations=$OPT_ITERATIONS --run-time=$OPT_RUN_TIME --sleep=$OPT_SLEEP --dest=$OPT_DEST --prefix=$OPT_PREFIX --log=$OPT_LOG --pid=$OPT_PID --plugin=$OPT_PLUGIN" + + log "Starting $0 $RAN_WITH" + choose_mongo_shell + MONGO_CONNECT_ARGS="$(mongo_connect_args)" + MONGO_MONITOR_ARGS="$(mongo_monitor_args)" + + mk_tmpdir + before_stalk + stalk + after_stalk + rm_tmpdir + remove_pid_file "$OPT_PID" + + log "Exiting because ${EXIT_REASON:-completed}" + [ "$OPT_COLLECT" ] && log "Files stored in $OPT_DEST" + log "$0 exit status $EXIT_STATUS" + exit "$EXIT_STATUS" +} + +if [ "${0##*/}" = "$TOOL" ] || [ "${0##*/}" = "bash" -a "${_:-""}" = "$0" ]; then + mk_tmpdir + parse_options "$0" "${@:-""}" + usage_or_errors "$0" + po_status=$? + rm_tmpdir + if [ "$po_status" -ne 0 ]; then + [ "$OPT_ERRS" -gt 0 ] && exit 1 + exit 0 + fi + + if ! set_trg_func "$OPT_FUNCTION"; then + option_error "Invalid --function value: $OPT_FUNCTION" + usage_or_errors "$0" >/dev/null || exit 1 + fi + + if [ "$OPT_PLUGIN" ]; then + if [ -f "$OPT_PLUGIN" ]; then + . "$OPT_PLUGIN" + else + die "Invalid --plugin value: $OPT_PLUGIN is not a file" + fi + fi + + if [ -n "$OPT_ASK_PASS" ]; then + stty_orig=$(stty -g) + echo -n "Enter MongoDB password: " + stty -echo + read OPT_PASSWORD + stty "$stty_orig" + echo + fi + + if [ -z "$OPT_STALK" ] && [ "$OPT_COLLECT" ]; then + OPT_CYCLES=0 + fi + + choose_mongo_shell + MONGO_CONNECT_ARGS="$(mongo_connect_args)" + MONGO_MONITOR_ARGS="$(mongo_monitor_args)" + + mongo_eval "db.getSiblingDB('admin').runCommand({ping: 1})" >/dev/null 2>&1 \ + || die "Cannot connect to MongoDB. Check that MongoDB is running and that the options are correct." + + if [ "$OPT_COLLECT" ]; then + [ -d "$OPT_DEST" ] || mkdir -p "$OPT_DEST" || die "Cannot make --dest $OPT_DEST" + ( + set -e + touch "$OPT_DEST/test" + rm "$OPT_DEST/test" + ) || die "Cannot read and write files to --dest $OPT_DEST" + fi + + if [ "$OPT_STALK" ] && [ "$OPT_DAEMONIZE" ]; then + touch "$OPT_LOG" || die "Cannot write to --log $OPT_LOG" + make_pid_file "$OPT_PID" $$ + main "${@:-""}" >"$OPT_LOG" 2>&1 & + echo "$!" > "$OPT_PID" + else + [ "$OPT_STALK" ] && make_pid_file "$OPT_PID" $$ + main "${@:-""}" + fi +fi + +:<<'DOCUMENTATION' +=pod + +=head1 NAME + +pt-mongodb-stalk - Collect forensic data about MongoDB when problems occur. + +=head1 SYNOPSIS + +Usage: pt-mongodb-stalk [OPTIONS] + +=head1 DESCRIPTION + +pt-mongodb-stalk watches a MongoDB server for a trigger condition and collects +diagnostic data when that trigger occurs. It follows the same basic operating +model as C, but uses MongoDB administration commands instead of +MySQL commands. + +The default trigger watches C. You can also +watch C and count active operations whose field matches a regular +expression. + +=head1 OPTIONS + +=over + +=item --ask-pass + +Prompt for a password when connecting to MongoDB. + +=item --authenticationDatabase + +type: string; default: admin + +Authentication database for the MongoDB shell connection. + +=item --collect + +default: yes; negatable: yes + +Collect diagnostic data when the trigger occurs. + +=item --config + +type: string + +Read this comma-separated list of config files. If specified, this must be the +first option on the command line. + +=item --cycles + +type: int; default: 5 + +How many times L<"--variable"> must be greater than L<"--threshold"> before +triggering L<"--collect">. + +=item --daemonize + +Daemonize the tool and log output to L<"--log">. + +=item --dest + +type: string; default: /var/lib/pt-mongodb-stalk + +Where to save diagnostic data. + +=item --disk-bytes-free + +type: size; default: 100M + +Do not collect if the disk has less than this much free space. + +=item --disk-pct-free + +type: int; default: 5 + +Do not collect if the disk has less than this percent free space. + +=item --function + +type: string; default: status + +Trigger source. Valid built-in values are C and C. + +With C, L<"--variable"> is a dot-separated path inside +C. Example: + + --function status --variable connections.current --threshold 200 + +With C, L<"--variable"> is a dot-separated field path inside each +C document and L<"--match"> is a regex. The trigger value is +the number of matching operations. Example: + + --function currentop --variable ns --match '^app\.orders$' --threshold 10 + +You can also specify a file that defines C. + +=item --help + +Print help and exit. + +=item --host + +short form: -h; type: string; default: localhost + +Host to connect to. + +=item --interval + +type: int; default: 1 + +How often to check the trigger, in seconds. + +=item --iterations + +type: int + +How many collections to perform before exiting. + +=item --log + +type: string; default: /var/log/pt-mongodb-stalk.log + +Print all output to this file when daemonized. + +=item --match + +type: string + +Regex pattern used with L<"--function"> C. + +=item --password + +short form: -p; type: string + +Password to use when connecting. + +=item --pid + +type: string; default: /var/run/pt-mongodb-stalk.pid + +Create the given PID file. + +=item --plugin + +type: string + +Load a plugin that defines any of the standard C or C +hooks. + +=item --port + +short form: -P; type: int; default: 27017 + +Port number to use for connection. + +=item --prefix + +type: string + +Filename prefix for diagnostic samples. + +=item --retention-count + +type: int; default: 0 + +Keep data for the last N runs. + +=item --retention-time + +type: int; default: 30 + +Number of days to retain collected samples. + +=item --run-time + +type: int; default: 30 + +How long interval collectors should run when the trigger occurs. + +=item --sleep + +type: int; default: 300 + +How long to sleep after collecting. + +=item --sleep-collect + +type: int; default: 1 + +Polling interval for interval collectors, in seconds. + +=item --stalk + +default: yes; negatable: yes + +Watch the server and wait for the trigger to occur. Specify C<--no-stalk> to +collect immediately. + +=item --threshold + +type: int; default: 100 + +Collection is triggered when L<"--variable"> is greater than this value. + +=item --tls + +default: ; negatable: yes + +Enable TLS for the MongoDB shell connection. + +=item --sslCAFile + +type: string + +Path to the TLS CA file. + +=item --sslPEMKeyFile + +type: string + +Path to the TLS client certificate and key file. + +=item --uri + +type: string + +Full MongoDB URI to connect with. + +=item --user + +short form: -u; type: string + +User for login. + +=item --variable + +type: string; default: connections.current + +Variable to watch inside C or C. + +=item --verbose + +type: int; default: 3 + +Print level of information. Values: 1 errors, 2 matching triggers and +collection info, 3 non-matching triggers. + +=item --version + +Print version and exit. + +=back + +=head1 EXAMPLES + +Run in stalking mode and collect twice when the trigger is met: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function status --variable connections.current --threshold 50 --cycles 3 --interval 1 --iterations 2 \ + --dest /tmp/pt-mongodb-stalk + +Run immediately without stalking and collect one short sample: + + pt-mongodb-stalk \ + --host localhost --port 30004 --user admin --password admin --authenticationDatabase admin \ + --no-stalk --iterations 1 --run-time 6 --sleep-collect 1 \ + --dest /tmp/pt-mongodb-stalk + +Run immediately without stalking and collect multiple short runs: + + pt-mongodb-stalk \ + --host localhost --port 30000 --user admin --password admin --authenticationDatabase admin \ + --no-stalk --iterations 3 --run-time 6 --sleep-collect 1 --sleep 1 \ + --dest /tmp/pt-mongodb-stalk + +Run immediately without stalking and collect fewer, more widely spaced samples: + + pt-mongodb-stalk \ + --host localhost --port 27000 --user admin --password admin --authenticationDatabase admin \ + --no-stalk --iterations 1 --run-time 10 --sleep-collect 2 \ + --dest /tmp/pt-mongodb-stalk + +Run in stalking mode using currentOp matches instead of a serverStatus metric: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function currentop --variable ns --match '^app\.orders$' --threshold 10 --cycles 2 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk + +=head1 COPYRIGHT, LICENSE, AND WARRANTY + +This program is copyright 2011-2026 Percona LLC and/or its affiliates. + +THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + +=cut +DOCUMENTATION diff --git a/bin/pt-mongodb-stalk.README b/bin/pt-mongodb-stalk.README new file mode 100644 index 000000000..01b6aec01 --- /dev/null +++ b/bin/pt-mongodb-stalk.README @@ -0,0 +1,172 @@ +pt-mongodb-stalk +================ + +Overview +-------- + +`pt-mongodb-stalk` is a Bash collector modeled after `pt-stalk`, adapted for +MongoDB. It watches a MongoDB node for a trigger condition and collects a small +runtime-focused sample set when the trigger fires, or collects immediately when +run with `--no-stalk`. + +The tool is intended for Linux systems and stores its output in a destination +directory as timestamped sample files plus a few fixed bookkeeping files. + + +What It Watches +--------------- + +`pt-mongodb-stalk` currently supports two built-in trigger sources: + +- `--function status` + Watches a dot-separated field inside `db.adminCommand({serverStatus: 1})`. + +- `--function currentop` + Counts matching active operations from + `db.adminCommand({currentOp: 1, active: true, idleConnections: false})`. + +Examples: + +```bash +pt-mongodb-stalk \ + --host localhost --port 27017 \ + --function status --variable connections.current --threshold 100 +``` + +```bash +pt-mongodb-stalk \ + --host localhost --port 27017 \ + --function currentop --variable ns --match '^app\.orders$' --threshold 10 +``` + + +How Collection Works +-------------------- + +When the trigger condition is met for the configured number of consecutive +cycles, the tool collects into `--dest`. + +Snapshot commands run once per collection iteration and are stored as +timestamped files. For example: + +- `2026_04_24_10_00_01-serverStatus.json` +- `2026_04_24_10_00_01-currentOp.json` +- `2026_04_24_10_00_01-ps.txt` + +Interval commands run once per collection iteration using `--sleep-collect` as +their polling interval and a count derived from `--run-time`. For example, +`--run-time 5 --sleep-collect 1` runs commands like `vmstat 1 5` and stores the +result in one timestamped file. + +The collection window is capped by `--run-time`. After a collection finishes, +the tool waits `--sleep` seconds before the next trigger check or collection +iteration. Collections do not overlap. + +The collector also writes a few fixed files in the destination directory: + +- `heartbeat` +- `log` +- `trigger` + + +MongoDB Data Collected +---------------------- + +Collected once per collection iteration: + +- `serverStatus` +- `currentOp` + +Collected once per collection iteration, using `--sleep-collect` as the polling +interval: + +- `mongostat` +- `mongotop` + +MongoDB JSON output from shell commands is cleaned before writing: + +- `ok` is removed +- `$clusterTime` is removed +- `operationTime` is removed + +Topology is still detected internally so the tool can decide whether the +process-specific `pidstat` file should be named `pidstat_mongod` or +`pidstat_mongos`, but no topology summary file is written. + + +System Data Collected +--------------------- + +Depending on tool availability, the collector also captures these snapshot +samples once per collection iteration: + +- `ps` using `ps faux` +- `pidstat_d` +- `pidstat_u` +- `pidstat_mongod` or `pidstat_mongos` + +These commands run once per collection iteration, using `--sleep-collect` as +their polling interval: + +- `vmstat` +- `iostat` +- `mpstat` +- `top` + +The tool may also copy one fixed file at the start of collection: + +- `messages.out` if `/var/log/messages` exists + + +Output Cleanup +-------------- + +At the end of a run: + +- zero-byte `.err` files are removed +- non-empty `.err` files are kept + +The collector still enforces disk-space safety checks, but it now uses internal +temporary files instead of writing disk-space snapshots into `--dest`. + + +Typical Test Runs +----------------- + +Collect immediately from a replica set member: + +```bash +pt-mongodb-stalk \ + --host localhost --port 27000 --user admin --password admin --authenticationDatabase admin \ + --no-stalk --iterations 1 --run-time 3 --sleep-collect 1 \ + --dest /tmp/pt-mongodb-stalk-rs27000 +``` + +Collect immediately from a `mongos` router: + +```bash +pt-mongodb-stalk \ + --host localhost --port 30000 --user admin --password admin --authenticationDatabase admin \ + --no-stalk --iterations 1 --run-time 3 --sleep-collect 1 \ + --dest /tmp/pt-mongodb-stalk-mongos30000 +``` + +Stalk until a `serverStatus` metric crosses a threshold: + +```bash +pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function status --variable connections.current --threshold 50 --cycles 3 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk-shard30001 --pid /tmp/pt-mongodb-stalk.pid +``` + + +Notes +----- + +- The tool is being developed for Linux use. +- Output directories can still grow quickly if `--run-time` is long and + `--sleep-collect` is short. +- A separate summary-style tool is expected to collect broader one-time server + and MongoDB metadata; `pt-mongodb-stalk` is now intentionally narrower and + focused on runtime sampling. From 478a9edaa3e0a3c33d4db4a5a74f9f2bf89d4175 Mon Sep 17 00:00:00 2001 From: Agustin Palazuelos Date: Wed, 6 May 2026 18:36:34 -0300 Subject: [PATCH 2/4] pt-mongodb-stalk tool --- bin/pt-mongodb-stalk | 221 +++++++++++++++++++++++++++++++++--- bin/pt-mongodb-stalk.README | 108 +++++++++++++++++- 2 files changed, 313 insertions(+), 16 deletions(-) diff --git a/bin/pt-mongodb-stalk b/bin/pt-mongodb-stalk index 216f5500c..83fa4b6cf 100755 --- a/bin/pt-mongodb-stalk +++ b/bin/pt-mongodb-stalk @@ -810,7 +810,7 @@ mongo_currentop_count() { fi mongo_eval " const __pt = db.getSiblingDB('admin'); - const __doc = __pt.runCommand({currentOp: 1, active: true, idleConnections: false}); + const __doc = __pt.runCommand({currentOp: 1, active: true, idleConnections: false, allUsers: true}); const __parts = '$field'.split('.'); const __re = ${js_regex}; let __count = 0; @@ -836,6 +836,56 @@ mongo_currentop_count() { " } +mongo_repl_lag_seconds() { + mongo_eval " + const __pt = db.getSiblingDB('admin'); + let __doc = {}; + try { + __doc = __pt.runCommand({replSetGetStatus: 1}); + } catch (e) { + print(''); + quit(0); + } + if (!__doc || !Array.isArray(__doc.members)) { + print(''); + quit(0); + } + + let __primary = null; + for (const __member of __doc.members) { + if (__member.stateStr === 'PRIMARY' && __member.optimeDate) { + __primary = __member; + break; + } + } + if (!__primary) { + print(0); + quit(0); + } + + const __primaryMs = new Date(__primary.optimeDate).getTime(); + let __maxLag = 0; + for (const __member of __doc.members) { + if (__member._id === __primary._id || !__member.optimeDate) { + continue; + } + const __state = __member.stateStr || ''; + if (!/^(SECONDARY|STARTUP2|RECOVERING|ROLLBACK)$/.test(__state)) { + continue; + } + const __memberMs = new Date(__member.optimeDate).getTime(); + if (isNaN(__memberMs)) { + continue; + } + const __lag = Math.max(0, Math.floor((__primaryMs - __memberMs) / 1000)); + if (__lag > __maxLag) { + __maxLag = __lag; + } + } + print(__maxLag); + " +} + _js_quote() { printf '%s' "$1" | perl -0pe 's/\\/\\\\/g; s/'"'"'/\\'"'"'/g; $_="'"'"'$_'"'"'"' } @@ -996,7 +1046,7 @@ collect_mongodb_data_one() { collect_mongodb_data_loop() { local d="$1" (mongo_admin_json "db.runCommand({serverStatus: 1})" > "$(sample_file "$d" "serverStatus" "json")") & - (mongo_admin_json "db.runCommand({currentOp: 1, active: true, idleConnections: false})" > "$(sample_file "$d" "currentOp" "json")") & + (mongo_admin_json "db.runCommand({currentOp: 1, active: true, idleConnections: false, allUsers: true})" > "$(sample_file "$d" "currentOp" "json")") & } collect_mongodb_data_two() { @@ -1091,10 +1141,28 @@ set_trg_func() { fi func=$(echo "$func" | tr '[:upper:]' '[:lower:]') - if [ "$func" = "status" ] || [ "$func" = "currentop" ]; then - TRIGGER_FUNCTION="trg_$func" - return 0 - fi + case "$func" in + status|currentop|cpu|memory|writewait|repllag) + TRIGGER_FUNCTION="trg_$func" + return 0 + ;; + mem) + TRIGGER_FUNCTION="trg_memory" + return 0 + ;; + processcpu|process-cpu|proc_cpu|proc-cpu) + TRIGGER_FUNCTION="trg_process_cpu" + return 0 + ;; + processmemory|process-memory|proc_memory|proc-memory) + TRIGGER_FUNCTION="trg_process_memory" + return 0 + ;; + replicationlag|replciationlag|waitforreplicationlag|waitforreplciationlag) + TRIGGER_FUNCTION="trg_repllag" + return 0 + ;; + esac return 1 } @@ -1109,6 +1177,92 @@ trg_currentop() { mongo_currentop_count "$var" } +trg_cpu() { + local cpu user nice system idle iowait irq softirq steal guest guest_nice + local idle_all non_idle total total_delta idle_delta value + local cpu_state last_total last_idle + + read -r cpu user nice system idle iowait irq softirq steal guest guest_nice < /proc/stat || return 1 + iowait=${iowait:-0} + irq=${irq:-0} + softirq=${softirq:-0} + steal=${steal:-0} + idle_all=$((idle + iowait)) + non_idle=$((user + nice + system + irq + softirq + steal)) + total=$((idle_all + non_idle)) + cpu_state="$PT_TMPDIR/cpu-trigger-state" + + if [ ! -f "$cpu_state" ]; then + echo "$total $idle_all" > "$cpu_state" + echo 0 + return 0 + fi + + read -r last_total last_idle < "$cpu_state" || { + echo "$total $idle_all" > "$cpu_state" + echo 0 + return 0 + } + total_delta=$((total - last_total)) + idle_delta=$((idle_all - last_idle)) + echo "$total $idle_all" > "$cpu_state" + + if [ "$total_delta" -le 0 ]; then + echo 0 + return 0 + fi + + value=$(awk -v total="$total_delta" -v idle="$idle_delta" 'BEGIN { printf "%.2f\n", ((total - idle) / total) * 100 }') + echo "$value" +} + +trg_memory() { + awk ' + /^MemTotal:/ { total = $2 } + /^MemAvailable:/ { available = $2 } + /^MemFree:/ { free = $2 } + /^Buffers:/ { buffers = $2 } + /^Cached:/ { cached = $2 } + END { + if (available == "" && free >= 0) { + available = free + buffers + cached + } + if (total > 0 && available >= 0) { + printf "%.2f\n", ((total - available) / total) * 100 + } + } + ' /proc/meminfo +} + +trg_process_cpu() { + local pid value + pid=$(get_mongo_pid) + [ -n "$pid" ] || return 1 + value=$(ps -p "$pid" -o %cpu= 2>/dev/null | awk '{gsub(/,/, "."); printf "%.2f\n", $1 + 0; exit}') + [ -n "$value" ] && echo "$value" +} + +trg_process_memory() { + local pid value + pid=$(get_mongo_pid) + [ -n "$pid" ] || return 1 + value=$(ps -p "$pid" -o %mem= 2>/dev/null | awk '{gsub(/,/, "."); printf "%.2f\n", $1 + 0; exit}') + [ -n "$value" ] && echo "$value" +} + +trg_writewait() { + local value + value=$(mongo_status_value "globalLock.currentQueue.writers") + if [ -z "$value" ]; then + value=0 + fi + echo "$value" +} + +trg_repllag() { + mongo_repl_lag_seconds +} + oktorun() { if [ "$OKTORUN" -eq 0 ]; then [ -z "$EXIT_REASON" ] && EXIT_REASON="OKTORUN is false" @@ -1372,8 +1526,8 @@ model as C, but uses MongoDB administration commands instead of MySQL commands. The default trigger watches C. You can also -watch C and count active operations whose field matches a regular -expression. +watch C, host CPU usage, host memory usage, queued writers, and +replica set replication lag. =head1 OPTIONS @@ -1435,7 +1589,8 @@ Do not collect if the disk has less than this percent free space. type: string; default: status -Trigger source. Valid built-in values are C and C. +Trigger source. Valid built-in values are C, C, C, +C, C, and C. With C, L<"--variable"> is a dot-separated path inside C. Example: @@ -1446,7 +1601,25 @@ With C, L<"--variable"> is a dot-separated field path inside each C document and L<"--match"> is a regex. The trigger value is the number of matching operations. Example: - --function currentop --variable ns --match '^app\.orders$' --threshold 10 + --function currentop --variable command.aggregate --match '^orders$' --threshold 10 + +With C, the trigger value is host CPU busy percentage from C. +With C, the trigger value is host memory used percentage from +C. C is accepted as an alias for C. Example: + + --function cpu --threshold 85 + --function memory --threshold 90 + +With C, the trigger value is +C. Example: + + --function writewait --threshold 5 + +With C, the trigger value is the maximum replica set member lag behind +the primary, in seconds, from C. C and +C are accepted as aliases. Example: + + --function repllag --threshold 30 You can also specify a file that defines C. @@ -1554,7 +1727,7 @@ collect immediately. =item --threshold -type: int; default: 100 +type: float; default: 100 Collection is triggered when L<"--variable"> is greater than this value. @@ -1592,7 +1765,8 @@ User for login. type: string; default: connections.current -Variable to watch inside C or C. +Variable to watch inside C or C. This option is ignored +by C, C, C, and C. =item --verbose @@ -1641,7 +1815,28 @@ Run in stalking mode using currentOp matches instead of a serverStatus metric: pt-mongodb-stalk \ --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ - --function currentop --variable ns --match '^app\.orders$' --threshold 10 --cycles 2 --interval 1 --iterations 1 \ + --function currentop --variable command.aggregate --match '^orders$' --threshold 10 --cycles 2 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk + +Run in stalking mode when host CPU is above 85 percent: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function cpu --threshold 85 --cycles 3 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk + +Run in stalking mode when host memory is above 90 percent: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function memory --threshold 90 --cycles 3 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk + +Run in stalking mode when replica set lag is above 30 seconds: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function repllag --threshold 30 --cycles 2 --interval 1 --iterations 1 \ --dest /tmp/pt-mongodb-stalk =head1 COPYRIGHT, LICENSE, AND WARRANTY diff --git a/bin/pt-mongodb-stalk.README b/bin/pt-mongodb-stalk.README index 01b6aec01..8fcc35fe4 100644 --- a/bin/pt-mongodb-stalk.README +++ b/bin/pt-mongodb-stalk.README @@ -16,14 +16,29 @@ directory as timestamped sample files plus a few fixed bookkeeping files. What It Watches --------------- -`pt-mongodb-stalk` currently supports two built-in trigger sources: +`pt-mongodb-stalk` currently supports these built-in trigger sources: - `--function status` Watches a dot-separated field inside `db.adminCommand({serverStatus: 1})`. - `--function currentop` Counts matching active operations from - `db.adminCommand({currentOp: 1, active: true, idleConnections: false})`. + `db.adminCommand({currentOp: 1, active: true, idleConnections: false, allUsers: true})`. + +- `--function cpu` + Watches host CPU busy percentage from `/proc/stat`. + +- `--function memory` + Watches host memory used percentage from `/proc/meminfo`. `mem` is accepted + as an alias. + +- `--function writewait` + Watches `serverStatus.globalLock.currentQueue.writers`. + +- `--function repllag` + Watches the maximum replica set member lag behind the primary, in seconds, + from `replSetGetStatus`. `replicationlag` and `waitForReplicationLag` are + accepted as aliases. Examples: @@ -36,7 +51,25 @@ pt-mongodb-stalk \ ```bash pt-mongodb-stalk \ --host localhost --port 27017 \ - --function currentop --variable ns --match '^app\.orders$' --threshold 10 + --function currentop --variable command.aggregate --match '^orders$' --threshold 10 +``` + +```bash +pt-mongodb-stalk \ + --host localhost --port 27017 \ + --function cpu --threshold 85 +``` + +```bash +pt-mongodb-stalk \ + --host localhost --port 27017 \ + --function memory --threshold 90 +``` + +```bash +pt-mongodb-stalk \ + --host localhost --port 27017 \ + --function repllag --threshold 30 ``` @@ -160,6 +193,75 @@ pt-mongodb-stalk \ --dest /tmp/pt-mongodb-stalk-shard30001 --pid /tmp/pt-mongodb-stalk.pid ``` +Stalk until host CPU, host memory, or replica set lag crosses a threshold: + +```bash +pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function cpu --threshold 85 --cycles 3 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk-shard30001 --pid /tmp/pt-mongodb-stalk.pid +``` + +```bash +pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function memory --threshold 90 --cycles 3 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk-shard30001 --pid /tmp/pt-mongodb-stalk.pid +``` + +```bash +pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function repllag --threshold 30 --cycles 2 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk-shard30001 --pid /tmp/pt-mongodb-stalk.pid +``` + + +Forcing Test Triggers +--------------------- + +`pt-mongodb-stalk-trigger-load` is a small helper for generating controlled +load while testing stalking mode. It should be run from another shell while +`pt-mongodb-stalk` is watching. + +Examples: + +```bash +pt-mongodb-stalk-trigger-load \ + --mode connections --host localhost --port 27000 --user admin --password admin \ + --connections 8 --duration 30 +``` + +```bash +pt-mongodb-stalk-trigger-load \ + --mode currentop --host localhost --port 27001 --user admin --password admin \ + --ops 2 --duration 30 --sleep-ms 1000 +``` + +```bash +pt-mongodb-stalk-trigger-load --mode cpu --duration 30 --cpu-workers 1 +``` + +```bash +pt-mongodb-stalk-trigger-load --mode memory --duration 30 --memory-mb 256 +``` + +```bash +pt-mongodb-stalk-trigger-load \ + --mode writewait --host localhost --port 27001 --user admin --password admin \ + --write-workers 2 --duration 30 +``` + +```bash +pt-mongodb-stalk-trigger-load \ + --mode repllag --host localhost --port 27000 --user admin --password admin \ + --write-workers 2 --duration 30 +``` + +The `writewait` and `repllag` modes generate write load, but whether they cross +a positive threshold depends on storage engine behavior, replica set speed, and +current cluster load. Use low test thresholds first. + Notes ----- From d099e322d4d34508a6cd23f787cfb1e2378ef8f5 Mon Sep 17 00:00:00 2001 From: Agustin Palazuelos Date: Thu, 14 May 2026 21:57:57 -0300 Subject: [PATCH 3/4] Address pt-mongodb-stalk review feedback --- bin/pt-mongodb-stalk | 76 +++++++ bin/pt-mongodb-stalk.README | 274 -------------------------- t/pt-mongodb-stalk/pt-mongodb-stalk.t | 181 +++++++++++++++++ 3 files changed, 257 insertions(+), 274 deletions(-) delete mode 100644 bin/pt-mongodb-stalk.README create mode 100644 t/pt-mongodb-stalk/pt-mongodb-stalk.t diff --git a/bin/pt-mongodb-stalk b/bin/pt-mongodb-stalk index 83fa4b6cf..43ff80549 100755 --- a/bin/pt-mongodb-stalk +++ b/bin/pt-mongodb-stalk @@ -1839,6 +1839,82 @@ Run in stalking mode when replica set lag is above 30 seconds: --function repllag --threshold 30 --cycles 2 --interval 1 --iterations 1 \ --dest /tmp/pt-mongodb-stalk +=head1 OUTPUT + +When the trigger condition is met for the configured number of consecutive +cycles, the tool collects into L<"--dest">. Snapshot commands run once per +collection iteration and are stored as timestamped files. For example: + + 2026_04_24_10_00_01-serverStatus.json + 2026_04_24_10_00_01-currentOp.json + 2026_04_24_10_00_01-ps.txt + +Interval commands run once per collection iteration using L<"--sleep-collect"> +as their polling interval and a count derived from L<"--run-time">. For +example, C<--run-time 5 --sleep-collect 1> runs commands like C +and stores the result in one timestamped file. + +The collection window is capped by L<"--run-time">. After a collection +finishes, the tool waits L<"--sleep"> seconds before the next trigger check or +collection iteration. Collections do not overlap. + +The collector also writes these fixed files in the destination directory: + + heartbeat + log + trigger + +=head1 COLLECTED DATA + +MongoDB data collected once per collection iteration: + + serverStatus + currentOp + +MongoDB interval tools collected once per collection iteration: + + mongostat + mongotop + +MongoDB JSON output from shell commands is cleaned before writing: C, +C<$clusterTime>, and C are removed. + +System data collected once per collection iteration, depending on tool +availability: + + ps faux + pidstat -d + pidstat -u + pidstat -urdwt for mongod or mongos + +System interval tools collected once per collection iteration, depending on +tool availability: + + vmstat + iostat + mpstat + top + +The process-specific C file is named C or +C according to the detected MongoDB process type. Topology +is detected internally for this purpose, but no topology summary file is +written. + +If C exists, the tool also copies it to C. + +=head1 OUTPUT CLEANUP + +At the end of a run, zero-byte C<.err> files are removed and non-empty C<.err> +files are kept. The collector enforces disk-space safety checks, but uses +internal temporary files instead of writing disk-space snapshots into +L<"--dest">. + +=head1 NOTES + +This tool is intended for Linux systems. A separate summary-style tool should +collect broader one-time server and MongoDB metadata; C is +focused on runtime sampling around a trigger event. + =head1 COPYRIGHT, LICENSE, AND WARRANTY This program is copyright 2011-2026 Percona LLC and/or its affiliates. diff --git a/bin/pt-mongodb-stalk.README b/bin/pt-mongodb-stalk.README deleted file mode 100644 index 8fcc35fe4..000000000 --- a/bin/pt-mongodb-stalk.README +++ /dev/null @@ -1,274 +0,0 @@ -pt-mongodb-stalk -================ - -Overview --------- - -`pt-mongodb-stalk` is a Bash collector modeled after `pt-stalk`, adapted for -MongoDB. It watches a MongoDB node for a trigger condition and collects a small -runtime-focused sample set when the trigger fires, or collects immediately when -run with `--no-stalk`. - -The tool is intended for Linux systems and stores its output in a destination -directory as timestamped sample files plus a few fixed bookkeeping files. - - -What It Watches ---------------- - -`pt-mongodb-stalk` currently supports these built-in trigger sources: - -- `--function status` - Watches a dot-separated field inside `db.adminCommand({serverStatus: 1})`. - -- `--function currentop` - Counts matching active operations from - `db.adminCommand({currentOp: 1, active: true, idleConnections: false, allUsers: true})`. - -- `--function cpu` - Watches host CPU busy percentage from `/proc/stat`. - -- `--function memory` - Watches host memory used percentage from `/proc/meminfo`. `mem` is accepted - as an alias. - -- `--function writewait` - Watches `serverStatus.globalLock.currentQueue.writers`. - -- `--function repllag` - Watches the maximum replica set member lag behind the primary, in seconds, - from `replSetGetStatus`. `replicationlag` and `waitForReplicationLag` are - accepted as aliases. - -Examples: - -```bash -pt-mongodb-stalk \ - --host localhost --port 27017 \ - --function status --variable connections.current --threshold 100 -``` - -```bash -pt-mongodb-stalk \ - --host localhost --port 27017 \ - --function currentop --variable command.aggregate --match '^orders$' --threshold 10 -``` - -```bash -pt-mongodb-stalk \ - --host localhost --port 27017 \ - --function cpu --threshold 85 -``` - -```bash -pt-mongodb-stalk \ - --host localhost --port 27017 \ - --function memory --threshold 90 -``` - -```bash -pt-mongodb-stalk \ - --host localhost --port 27017 \ - --function repllag --threshold 30 -``` - - -How Collection Works --------------------- - -When the trigger condition is met for the configured number of consecutive -cycles, the tool collects into `--dest`. - -Snapshot commands run once per collection iteration and are stored as -timestamped files. For example: - -- `2026_04_24_10_00_01-serverStatus.json` -- `2026_04_24_10_00_01-currentOp.json` -- `2026_04_24_10_00_01-ps.txt` - -Interval commands run once per collection iteration using `--sleep-collect` as -their polling interval and a count derived from `--run-time`. For example, -`--run-time 5 --sleep-collect 1` runs commands like `vmstat 1 5` and stores the -result in one timestamped file. - -The collection window is capped by `--run-time`. After a collection finishes, -the tool waits `--sleep` seconds before the next trigger check or collection -iteration. Collections do not overlap. - -The collector also writes a few fixed files in the destination directory: - -- `heartbeat` -- `log` -- `trigger` - - -MongoDB Data Collected ----------------------- - -Collected once per collection iteration: - -- `serverStatus` -- `currentOp` - -Collected once per collection iteration, using `--sleep-collect` as the polling -interval: - -- `mongostat` -- `mongotop` - -MongoDB JSON output from shell commands is cleaned before writing: - -- `ok` is removed -- `$clusterTime` is removed -- `operationTime` is removed - -Topology is still detected internally so the tool can decide whether the -process-specific `pidstat` file should be named `pidstat_mongod` or -`pidstat_mongos`, but no topology summary file is written. - - -System Data Collected ---------------------- - -Depending on tool availability, the collector also captures these snapshot -samples once per collection iteration: - -- `ps` using `ps faux` -- `pidstat_d` -- `pidstat_u` -- `pidstat_mongod` or `pidstat_mongos` - -These commands run once per collection iteration, using `--sleep-collect` as -their polling interval: - -- `vmstat` -- `iostat` -- `mpstat` -- `top` - -The tool may also copy one fixed file at the start of collection: - -- `messages.out` if `/var/log/messages` exists - - -Output Cleanup --------------- - -At the end of a run: - -- zero-byte `.err` files are removed -- non-empty `.err` files are kept - -The collector still enforces disk-space safety checks, but it now uses internal -temporary files instead of writing disk-space snapshots into `--dest`. - - -Typical Test Runs ------------------ - -Collect immediately from a replica set member: - -```bash -pt-mongodb-stalk \ - --host localhost --port 27000 --user admin --password admin --authenticationDatabase admin \ - --no-stalk --iterations 1 --run-time 3 --sleep-collect 1 \ - --dest /tmp/pt-mongodb-stalk-rs27000 -``` - -Collect immediately from a `mongos` router: - -```bash -pt-mongodb-stalk \ - --host localhost --port 30000 --user admin --password admin --authenticationDatabase admin \ - --no-stalk --iterations 1 --run-time 3 --sleep-collect 1 \ - --dest /tmp/pt-mongodb-stalk-mongos30000 -``` - -Stalk until a `serverStatus` metric crosses a threshold: - -```bash -pt-mongodb-stalk \ - --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ - --function status --variable connections.current --threshold 50 --cycles 3 --interval 1 --iterations 1 \ - --dest /tmp/pt-mongodb-stalk-shard30001 --pid /tmp/pt-mongodb-stalk.pid -``` - -Stalk until host CPU, host memory, or replica set lag crosses a threshold: - -```bash -pt-mongodb-stalk \ - --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ - --function cpu --threshold 85 --cycles 3 --interval 1 --iterations 1 \ - --dest /tmp/pt-mongodb-stalk-shard30001 --pid /tmp/pt-mongodb-stalk.pid -``` - -```bash -pt-mongodb-stalk \ - --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ - --function memory --threshold 90 --cycles 3 --interval 1 --iterations 1 \ - --dest /tmp/pt-mongodb-stalk-shard30001 --pid /tmp/pt-mongodb-stalk.pid -``` - -```bash -pt-mongodb-stalk \ - --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ - --function repllag --threshold 30 --cycles 2 --interval 1 --iterations 1 \ - --dest /tmp/pt-mongodb-stalk-shard30001 --pid /tmp/pt-mongodb-stalk.pid -``` - - -Forcing Test Triggers ---------------------- - -`pt-mongodb-stalk-trigger-load` is a small helper for generating controlled -load while testing stalking mode. It should be run from another shell while -`pt-mongodb-stalk` is watching. - -Examples: - -```bash -pt-mongodb-stalk-trigger-load \ - --mode connections --host localhost --port 27000 --user admin --password admin \ - --connections 8 --duration 30 -``` - -```bash -pt-mongodb-stalk-trigger-load \ - --mode currentop --host localhost --port 27001 --user admin --password admin \ - --ops 2 --duration 30 --sleep-ms 1000 -``` - -```bash -pt-mongodb-stalk-trigger-load --mode cpu --duration 30 --cpu-workers 1 -``` - -```bash -pt-mongodb-stalk-trigger-load --mode memory --duration 30 --memory-mb 256 -``` - -```bash -pt-mongodb-stalk-trigger-load \ - --mode writewait --host localhost --port 27001 --user admin --password admin \ - --write-workers 2 --duration 30 -``` - -```bash -pt-mongodb-stalk-trigger-load \ - --mode repllag --host localhost --port 27000 --user admin --password admin \ - --write-workers 2 --duration 30 -``` - -The `writewait` and `repllag` modes generate write load, but whether they cross -a positive threshold depends on storage engine behavior, replica set speed, and -current cluster load. Use low test thresholds first. - - -Notes ------ - -- The tool is being developed for Linux use. -- Output directories can still grow quickly if `--run-time` is long and - `--sleep-collect` is short. -- A separate summary-style tool is expected to collect broader one-time server - and MongoDB metadata; `pt-mongodb-stalk` is now intentionally narrower and - focused on runtime sampling. diff --git a/t/pt-mongodb-stalk/pt-mongodb-stalk.t b/t/pt-mongodb-stalk/pt-mongodb-stalk.t new file mode 100644 index 000000000..f1a714a7d --- /dev/null +++ b/t/pt-mongodb-stalk/pt-mongodb-stalk.t @@ -0,0 +1,181 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use File::Path qw(make_path); +use File::Temp qw(tempdir); +use Test::More; + +use PerconaTest; + +my $tool = "$trunk/bin/pt-mongodb-stalk"; +my $tmpdir = tempdir("pt-mongodb-stalk.XXXXXX", TMPDIR => 1, CLEANUP => 1); +my $fakebin = "$tmpdir/bin"; +my $run_no = 0; + +make_path($fakebin); + +sub write_fake_cmd { + my ($name, $body) = @_; + my $file = "$fakebin/$name"; + open my $fh, ">", $file or die "Cannot write $file: $OS_ERROR"; + print {$fh} $body; + close $fh; + chmod 0755, $file or die "Cannot chmod $file: $OS_ERROR"; + return $file; +} + +sub run_capture { + my (@cmd) = @_; + $run_no++; + my $out = "$tmpdir/run-$run_no.out"; + my $pid = fork(); + die "Cannot fork: $OS_ERROR" unless defined $pid; + + if ( !$pid ) { + open STDOUT, ">", $out or die "Cannot redirect stdout: $OS_ERROR"; + open STDERR, ">&", \*STDOUT or die "Cannot redirect stderr: $OS_ERROR"; + exec @cmd; + die "Cannot exec $cmd[0]: $OS_ERROR"; + } + + waitpid($pid, 0); + my $status = $CHILD_ERROR >> 8; + my $text = slurp_file($out); + return ($status, $text); +} + +sub glob_count { + my ($pattern) = @_; + my @files = glob($pattern); + return scalar @files; +} + +write_fake_cmd("mongosh", <<'SH'); +#!/usr/bin/env bash +[ -n "${MONGOSH_LOG:-}" ] && printf '%s\n' "$*" >> "$MONGOSH_LOG" +case "$*" in + *"__serverStatus.process"*) + printf 'mongod\tshard\tPRIMARY\trs0\n' + ;; + *"currentOp: 1"*) + printf '{"inprog":[]}\n' + ;; + *"serverStatus: 1"*) + printf '{"process":"mongod","connections":{"current":5},"globalLock":{"currentQueue":{"writers":0}}}\n' + ;; + *"replSetGetStatus: 1"*) + printf '{"members":[]}\n' + ;; +esac +exit 0 +SH + +write_fake_cmd("mongostat", <<'SH'); +#!/usr/bin/env bash +echo "insert query update delete getmore command dirty used flushes vsize res qrw arw net_in net_out conn time" +echo "0 0 0 0 0 1|0 0.0% 1.0% 0 100M 50M 0|0 0|0 1k 2k 1 00:00:00" +echo "2026-05-14T00:00:00.000+0000 connected to: mongodb://localhost:27017/" >&2 +SH + +write_fake_cmd("mongotop", <<'SH'); +#!/usr/bin/env bash +echo "ns total read write" +echo "admin.system.version 1ms 1ms 0ms" +echo "2026-05-14T00:00:00.000+0000 connected to: mongodb://localhost:27017/" >&2 +SH + +write_fake_cmd("df", <<'SH'); +#!/usr/bin/env bash +cat </dev/null`); + +ok(-f "$dest/trigger", "creates trigger file"); +ok(-f "$dest/heartbeat", "creates heartbeat file"); +ok(-f "$dest/log", "creates log file"); + +is(glob_count("$dest/*-serverStatus.json"), 1, "collects serverStatus once"); +is(glob_count("$dest/*-currentOp.json"), 1, "collects currentOp once"); +is(glob_count("$dest/*-mongostat.txt"), 1, "collects mongostat once"); +is(glob_count("$dest/*-mongotop.txt"), 1, "collects mongotop once"); +is(glob_count("$dest/*-ps.txt"), 1, "collects ps once"); +is(glob_count("$dest/*-pidstat_mongod.txt"), 1, "collects process pidstat with mongod label"); + +my @mongotop_err = glob("$dest/*-mongotop.err"); +is(scalar @mongotop_err, 1, "keeps non-empty mongotop stderr"); +like(slurp_file($mongotop_err[0]), qr/connected to: mongodb:\/\/localhost:27017/, "mongotop stderr keeps connection banner"); + +my $mongosh_log = slurp_file($ENV{MONGOSH_LOG}); +like( + $mongosh_log, + qr/currentOp: 1, active: true, idleConnections: false, allUsers: true/, + "currentOp command includes allUsers" +); + +ok(!-e "$dest/topology", "does not write topology file"); +ok(!-e "$dest/disk-space", "does not write disk-space file"); + +done_testing; From 9c52ed5d7041c10f9a484105e9bc2bbdf25b67c4 Mon Sep 17 00:00:00 2001 From: Agustin Palazuelos Date: Fri, 15 May 2026 16:37:07 -0300 Subject: [PATCH 4/4] Clarify pt-mongodb-stalk embedded section comments --- bin/pt-mongodb-stalk | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/pt-mongodb-stalk b/bin/pt-mongodb-stalk index 43ff80549..71f0e2433 100755 --- a/bin/pt-mongodb-stalk +++ b/bin/pt-mongodb-stalk @@ -7,8 +7,8 @@ set -u # ########################################################################### -# log_warn_die package -# This package is a copy without comments from the original. The original +# log_warn_die section +# This section is adapted from the original. The original # with comments and its test file can be found in the GitHub repository at, # lib/bash/log_warn_die.sh # t/lib/bash/log_warn_die.sh @@ -48,12 +48,12 @@ _d () { } # ########################################################################### -# End log_warn_die package +# End log_warn_die section # ########################################################################### # ########################################################################### -# subshell package -# This package is a copy without comments from the original. The original +# subshell section +# This section is adapted from the original. The original # with comments and its test file can be found in the GitHub repository at, # lib/bash/subshell.sh # t/lib/bash/subshell.sh @@ -96,12 +96,12 @@ kill_all_subshells() { } # ########################################################################### -# End subshell package +# End subshell section # ########################################################################### # ########################################################################### -# parse_options package -# This package is a copy without comments from the original. The original +# parse_options section +# This section is adapted from the original. The original # with comments and its test file can be found in the GitHub repository at, # lib/bash/parse_options.sh # t/lib/bash/parse_options.sh @@ -474,7 +474,7 @@ size_to_bytes() { } # ########################################################################### -# tmpdir package +# tmpdir section # ########################################################################### PT_TMPDIR="" @@ -505,7 +505,7 @@ remove_empty_err_files() { } # ########################################################################### -# alt_cmds package +# alt_cmds section # ########################################################################### _which() { @@ -526,7 +526,7 @@ _pidof() { } # ########################################################################### -# safeguards package +# safeguards section # ########################################################################### disk_space() { @@ -568,7 +568,7 @@ check_disk_space() { } # ########################################################################### -# daemon package +# daemon section # ########################################################################### make_pid_file() { @@ -1007,7 +1007,7 @@ collect_command_sample() { } # ########################################################################### -# collection package +# collection section # ########################################################################### collect() {