diff --git a/bin/pt-mongodb-stalk b/bin/pt-mongodb-stalk new file mode 100755 index 000000000..71f0e2433 --- /dev/null +++ b/bin/pt-mongodb-stalk @@ -0,0 +1,1927 @@ +#!/usr/bin/env bash + +# This program is part of Percona Toolkit: http://www.percona.com/software/ +# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal +# notices and disclaimers. + +set -u + +# ########################################################################### +# log_warn_die section +# This section is adapted from the original. The original +# with comments and its test file can be found in the GitHub repository at, +# lib/bash/log_warn_die.sh +# t/lib/bash/log_warn_die.sh +# See https://github.com/percona/percona-toolkit for more information. +# ########################################################################### + +PTFUNCNAME="" +PTDEBUG="${PTDEBUG:-""}" +EXIT_STATUS=0 + +ts() { + TS=$(date +%F-%T | tr ':-' '_') + echo "$TS $*" +} + +info() { + [ "${OPT_VERBOSE:-3}" -ge 3 ] && ts "$*" +} + +log() { + [ "${OPT_VERBOSE:-3}" -ge 2 ] && ts "$*" +} + +warn() { + [ "${OPT_VERBOSE:-3}" -ge 1 ] && ts "$*" >&2 + EXIT_STATUS=1 +} + +die() { + ts "$*" >&2 + EXIT_STATUS=1 + exit 1 +} + +_d () { + [ "$PTDEBUG" ] && echo "# $PTFUNCNAME: $(ts "$*")" >&2 +} + +# ########################################################################### +# End log_warn_die section +# ########################################################################### + +# ########################################################################### +# subshell section +# This section is adapted from the original. The original +# with comments and its test file can be found in the GitHub repository at, +# lib/bash/subshell.sh +# t/lib/bash/subshell.sh +# See https://github.com/percona/percona-toolkit for more information. +# ########################################################################### + + +wait_for_subshells() { + local max_wait=$1 + if [ "$(jobs)" ]; then + log "Waiting up to $max_wait seconds for subprocesses to finish..." + local slept=0 + while [ -n "$(jobs)" ]; do + local subprocess_still_running="" + for pid in $(jobs -p); do + if kill -0 "$pid" >/dev/null 2>&1; then + subprocess_still_running=1 + fi + done + if [ "$subprocess_still_running" ]; then + sleep 1 + slept=$((slept + 1)) + [ "$slept" -ge "$max_wait" ] && break + else + break + fi + done + fi +} + +kill_all_subshells() { + if [ "$(jobs)" ]; then + for pid in $(jobs -p); do + if kill -0 "$pid" >/dev/null 2>&1; then + log "Killing subprocess $pid" + kill "$pid" >/dev/null 2>&1 + fi + done + fi +} + +# ########################################################################### +# End subshell section +# ########################################################################### + +# ########################################################################### +# parse_options section +# This section is adapted from the original. The original +# with comments and its test file can be found in the GitHub repository at, +# lib/bash/parse_options.sh +# t/lib/bash/parse_options.sh +# See https://github.com/percona/percona-toolkit for more information. +# ########################################################################### + +ARGV="" # Non-option args (probably input files) +EXT_ARGV="" # Everything after -- (args for an external command) +HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV +OPT_ERRS=0 # How many command line option errors +OPT_VERSION="" # If --version was specified +OPT_HELP="" # If --help was specified +OPT_ASK_PASS="" # If --ask-pass was specified +PO_DIR="" # Directory with program option spec files +GLOBAL_CONFIG=0 # We ignore non-recognized options in global configs + +usage() { + local file="$1" + + local usage="$(grep '^Usage: ' "$file")" + echo "$usage" + echo + echo "For more information, 'man $TOOL' or 'perldoc $file'." +} + +usage_or_errors() { + local file="$1" + + if [ "$OPT_VERSION" ]; then + local version + version=$(grep '^pt-[^ ]\+ [0-9]' "$file") + echo "$version" + return 1 + fi + + if [ -z "$(_which perl)" ]; then + echo "Perl binary required to run this tool" + return 1 + fi + + if [ "$OPT_HELP" ]; then + usage "$file" + echo + echo "Command line options:" + echo + perl -e ' + use strict; + use warnings FATAL => qw(all); + my $lcol = 24; + my $rcol = 80 - $lcol; + my $name; + while ( <> ) { + my $line = $_; + chomp $line; + if ( $line =~ s/^long:/ --/ ) { + $name = $line; + } + elsif ( $line =~ s/^desc:// ) { + $line =~ s/ +$//mg; + my @lines = grep { $_ } + $line =~ m/(.{0,$rcol})(?:\s+|\Z)/g; + if ( length($name) >= $lcol ) { + print $name, "\n", (q{ } x $lcol); + } + else { + printf "%-${lcol}s", $name; + } + print join("\n" . (q{ } x $lcol), @lines); + print "\n"; + } + } + ' "$PO_DIR"/* + echo + echo "Examples:" + echo + perl -e ' + use strict; + use warnings FATAL => qw(all); + my $in = 0; + while ( <> ) { + if ( /^=head1 EXAMPLES/ ) { + $in = 1; + next; + } + last if $in && /^=head1 /; + next unless $in; + next if /^=cut/; + next if /^=over/; + next if /^=back/; + next if /^=pod/; + s/^C<([^>]+)>/$1/g; + s/L<"[^"]+">//g; + print; + } + ' "$file" + return 1 + fi + + if [ "$OPT_ERRS" -gt 0 ]; then + echo + usage "$file" + return 1 + fi + + return 0 +} + +option_error() { + local err="$1" + OPT_ERRS=$((OPT_ERRS + 1)) + echo "$err" >&2 +} + +parse_options() { + local file="$1" + shift + + ARGV="" + EXT_ARGV="" + HAVE_EXT_ARGV="" + OPT_ERRS=0 + OPT_VERSION="" + OPT_HELP="" + OPT_ASK_PASS="" + PO_DIR="$PT_TMPDIR/po" + + [ -d "$PO_DIR" ] || mkdir "$PO_DIR" || die "Cannot mkdir $PO_DIR" + rm -rf "$PO_DIR"/* || die "Cannot rm -rf $PO_DIR/*" + + _parse_pod "$file" + _eval_po + + if [ $# -ge 2 ] && [ "$1" = "--config" ]; then + shift + local user_config_files="$1" + shift + local IFS="," + for user_config_file in $user_config_files; do + _parse_config_files "$user_config_file" + done + else + GLOBAL_CONFIG=1 + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" + GLOBAL_CONFIG=0 + _parse_config_files "/etc/percona-toolkit/$TOOL.conf" + if [ "${HOME:-}" ]; then + GLOBAL_CONFIG=1 + _parse_config_files "$HOME/.percona-toolkit.conf" + GLOBAL_CONFIG=0 + _parse_config_files "$HOME/.$TOOL.conf" + fi + fi + + _parse_command_line "${@:-""}" +} + +_parse_pod() { + local file="$1" + + PO_FILE="$file" PO_DIR="$PO_DIR" perl -e ' + $/ = ""; + my $file = $ENV{PO_FILE}; + open my $fh, "<", $file or die "Cannot open $file: $!"; + while ( defined(my $para = <$fh>) ) { + next unless $para =~ m/^=head1 OPTIONS/; + while ( defined(my $para = <$fh>) ) { + last if $para =~ m/^=head1/; + chomp; + if ( $para =~ m/^=item --(\S+)/ ) { + my $opt = $1; + my $file = "$ENV{PO_DIR}/$opt"; + open my $opt_fh, ">", $file or die "Cannot open $file: $!"; + print $opt_fh "long:$opt\n"; + $para = <$fh>; + chomp; + if ( $para =~ m/^[a-z ]+:/ ) { + map { + chomp; + my ($attrib, $val) = split(/: /, $_); + print $opt_fh "$attrib:$val\n"; + } split(/; /, $para); + $para = <$fh>; + chomp; + } + my ($desc) = $para =~ m/^([^?.]+)/; + print $opt_fh "desc:$desc.\n"; + close $opt_fh; + } + } + last; + } + ' +} + +_eval_po() { + local IFS=":" + for opt_spec in "$PO_DIR"/*; do + local opt="" + local default_val="" + local neg=0 + local size=0 + while read -r key val; do + case "$key" in + long) + opt=$(echo "$val" | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]') + ;; + default) + default_val="$val" + ;; + "short form"|desc) + ;; + type) + [ "$val" = "size" ] && size=1 + ;; + negatable) + [ "$val" = "yes" ] && neg=1 + ;; + *) + ;; + esac + done < "$opt_spec" + + [ -n "$opt" ] || die "No long attribute in option spec $opt_spec" + + if [ "$size" -eq 1 ] && [ -n "$default_val" ]; then + default_val=$(size_to_bytes "$default_val") + fi + + eval "OPT_${opt}"='"$default_val"' + if [ "$neg" -eq 1 ] && [ "$default_val" = "yes" ]; then + : + fi + done +} + +_parse_config_files() { + for config_file in "${@:-""}"; do + test -f "$config_file" || continue + while read -r config_opt; do + echo "$config_opt" | grep '^[ ]*[^#]' >/dev/null 2>&1 || continue + config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/\s[ ]*#.*$//')" + [ "$config_opt" = "" ] && continue + if ! [ "$HAVE_EXT_ARGV" ]; then + config_opt="--$config_opt" + fi + _parse_command_line "$config_opt" + done < "$config_file" + HAVE_EXT_ARGV="" + done +} + +_parse_command_line() { + local opt="" + local val="" + local next_opt_is_val="" + local opt_is_ok="" + local opt_is_negated="" + local real_opt="" + local required_arg="" + local spec="" + + for opt in "${@:-""}"; do + if [ "$opt" = "--" -o "$opt" = "----" ]; then + HAVE_EXT_ARGV=1 + continue + fi + if [ "$HAVE_EXT_ARGV" ]; then + if [ "$EXT_ARGV" ]; then + EXT_ARGV="$EXT_ARGV $opt" + else + EXT_ARGV="$opt" + fi + continue + fi + + if [ "$next_opt_is_val" ]; then + next_opt_is_val="" + if [ $# -eq 0 ] || [ "$(expr "$opt" : "\-")" -eq 1 ]; then + option_error "$real_opt requires a $required_arg argument" + continue + fi + val="$opt" + opt_is_ok=1 + else + if [ "$(expr "$opt" : "\-")" -eq 0 ]; then + ARGV="${ARGV:+$ARGV }$opt" + continue + fi + + real_opt="$opt" + if $(echo "$opt" | grep '^--no[^-]' >/dev/null); then + local base_opt + base_opt=$(echo "$opt" | sed 's/^--no//') + if [ -f "$PT_TMPDIR/po/$base_opt" ]; then + opt_is_negated=1 + opt="$base_opt" + else + opt=$(echo "$opt" | sed 's/^-*//') + fi + else + if $(echo "$opt" | grep '^--no-' >/dev/null); then + opt_is_negated=1 + opt=$(echo "$opt" | sed 's/^--no-//') + else + opt=$(echo "$opt" | sed 's/^-*//') + fi + fi + + if $(echo "$opt" | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then + val="$(echo "$opt" | awk '{ st = index($0,"="); print substr($0, st+1)}')" + opt="$(echo "$opt" | awk -F= '{print $1}')" + fi + + if [ -f "$PT_TMPDIR/po/$opt" ]; then + spec="$PT_TMPDIR/po/$opt" + else + spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* 2>/dev/null | cut -d ':' -f 1) + if [ -z "$spec" ]; then + if [ "$GLOBAL_CONFIG" -eq 1 ]; then + continue + else + option_error "Unknown option: $real_opt" + continue + fi + fi + fi + + required_arg=$(awk -F: '/^type:/{print $2}' "$spec") + if [ "$required_arg" ]; then + if [ "$val" ]; then + opt_is_ok=1 + else + next_opt_is_val=1 + fi + else + if [ "$val" ]; then + option_error "Option $real_opt does not take a value" + continue + fi + if [ "$opt_is_negated" ]; then + val="" + else + val="yes" + fi + opt_is_ok=1 + fi + fi + + if [ "$opt_is_ok" ]; then + opt=$(grep '^long:' "$spec" | cut -d':' -f2 | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]') + if grep "^type:size" "$spec" >/dev/null; then + val=$(size_to_bytes "$val") + fi + eval "OPT_$opt"='$val' + opt="" + val="" + next_opt_is_val="" + opt_is_ok="" + opt_is_negated="" + real_opt="" + required_arg="" + spec="" + fi + done +} + +size_to_bytes() { + local size="$1" + echo "$size" | perl -ne '%f=(B=>1, K=>1_024, M=>1_048_576, G=>1_073_741_824, T=>1_099_511_627_776); m/^(\d+)([kMGT])?/i; print $1 * $f{uc($2 || "B")};' +} + +# ########################################################################### +# tmpdir section +# ########################################################################### + +PT_TMPDIR="" + +mk_tmpdir() { + local dir="${1:-""}" + if [ -n "$dir" ]; then + [ -d "$dir" ] || mkdir "$dir" || die "Cannot make tmpdir $dir" + PT_TMPDIR="$dir" + else + local tool="${0##*/}" + local pid="$$" + PT_TMPDIR=$(mktemp -d -t "${tool}.${pid}.XXXXXX") || die "Cannot make secure tmpdir" + fi +} + +rm_tmpdir() { + if [ -n "$PT_TMPDIR" ] && [ -d "$PT_TMPDIR" ]; then + rm -rf "$PT_TMPDIR" + fi + PT_TMPDIR="" +} + +remove_empty_err_files() { + local dir="$1" + [ -d "$dir" ] || return 0 + find "$dir" -maxdepth 1 -type f -name '*.err' -size 0 -delete 2>/dev/null || true +} + +# ########################################################################### +# alt_cmds section +# ########################################################################### + +_which() { + if [ -x /usr/bin/which ]; then + /usr/bin/which "$1" 2>/dev/null | awk '{print $1}' + elif which which 1>/dev/null 2>&1; then + which "$1" 2>/dev/null | awk '{print $1}' + else + echo "$1" + fi +} + +_pidof() { + local cmd="$1" + if ! pidof "$cmd" 2>/dev/null; then + ps -eo pid,ucomm | awk -v comm="$cmd" '$2 == comm { print $1 }' + fi +} + +# ########################################################################### +# safeguards section +# ########################################################################### + +disk_space() { + local filesystem="${1:-$PWD}" + df -P -k "$filesystem" +} + +check_disk_space() { + local file="$1" + local min_free_bytes="${2:-0}" + local min_free_pct="${3:-0}" + local bytes_margin="${4:-0}" + + local used_bytes free_bytes pct_used pct_free real_free_bytes real_pct_free + used_bytes=$(tail -n 1 "$file" | perl -ane 'print $F[2] * 1024') + free_bytes=$(tail -n 1 "$file" | perl -ane 'print $F[3] * 1024') + pct_used=$(tail -n 1 "$file" | perl -ane 'print ($F[4] =~ m/(\d+)/)') + pct_free=$((100 - pct_used)) + real_free_bytes=$free_bytes + real_pct_free=$pct_free + + if [ "$bytes_margin" -gt 0 ]; then + used_bytes=$((used_bytes + bytes_margin)) + free_bytes=$((free_bytes - bytes_margin)) + pct_used=$(perl -e "print int(($used_bytes/($used_bytes + $free_bytes)) * 100)") + pct_free=$((100 - pct_used)) + fi + + if [ "$free_bytes" -lt "$min_free_bytes" ] || [ "$pct_free" -lt "$min_free_pct" ]; then + warn "Not enough free disk space: + Limit: ${min_free_pct}% free, ${min_free_bytes} bytes free + Actual: ${real_pct_free}% free, ${real_free_bytes} bytes free (- $bytes_margin bytes margin) +" + cat "$file" >&2 + return 1 + fi + + return 0 +} + +# ########################################################################### +# daemon section +# ########################################################################### + +make_pid_file() { + local file="$1" + local pid="$2" + + if [ -f "$file" ]; then + local old_pid + old_pid=$(cat "$file") + if [ -z "$old_pid" ]; then + die "PID file $file already exists but it is empty" + else + kill -0 "$old_pid" 2>/dev/null + if [ $? -eq 0 ]; then + die "PID file $file already exists and its PID ($old_pid) is running" + fi + fi + fi + + echo "$pid" > "$file" || die "Cannot create or write PID file $file" +} + +remove_pid_file() { + local file="$1" + [ -f "$file" ] && rm "$file" +} + +# ########################################################################### +# Mongo helpers +# ########################################################################### + +CMD_MONGOSH="${CMD_MONGOSH:-"$(_which mongosh)"}" +CMD_MONGO="${CMD_MONGO:-"$(_which mongo)"}" +CMD_MONGOSTAT="${CMD_MONGOSTAT:-"$(_which mongostat)"}" +CMD_MONGOTOP="${CMD_MONGOTOP:-"$(_which mongotop)"}" +CMD_IOSTAT="${CMD_IOSTAT:-"$(_which iostat)"}" +CMD_VMSTAT="${CMD_VMSTAT:-"$(_which vmstat)"}" +CMD_MPSTAT="${CMD_MPSTAT:-"$(_which mpstat)"}" +CMD_PIDSTAT="${CMD_PIDSTAT:-"$(_which pidstat)"}" +CMD_TOP="${CMD_TOP:-"$(_which top)"}" +CMD_PMAP="${CMD_PMAP:-"$(_which pmap)"}" +CMD_DMESG="${CMD_DMESG:-"$(_which dmesg)"}" +CMD_JOURNALCTL="${CMD_JOURNALCTL:-"$(_which journalctl)"}" + +MONGO_SHELL="" +MONGO_EVAL_PREFIX="" +MONGO_AUTH_DB="" +MONGO_CONNECT_ARGS="" +MONGO_MONITOR_ARGS="" +MONGO_REPLSET_NAME="" +MONGO_REPLICA_SET_ROLE="" +MONGO_PROCESS_TYPE="" +MONGO_CLUSTER_ROLE="" + +choose_mongo_shell() { + if [ -n "$CMD_MONGOSH" ]; then + MONGO_SHELL="$CMD_MONGOSH" + MONGO_EVAL_PREFIX="EJSON.stringify" + elif [ -n "$CMD_MONGO" ]; then + MONGO_SHELL="$CMD_MONGO" + MONGO_EVAL_PREFIX="JSON.stringify" + else + die "Cannot execute mongosh or mongo. Check that one is in PATH." + fi +} + +mongo_connect_args() { + local args="" + if [ -n "$OPT_URI" ]; then + args="$OPT_URI" + else + args="mongodb://${OPT_HOST}:${OPT_PORT}/admin" + fi + + if [ -n "$OPT_USER" ]; then + args="$args --username=$OPT_USER" + fi + if [ -n "$OPT_PASSWORD" ]; then + args="$args --password=$OPT_PASSWORD" + fi + if [ -n "$OPT_AUTHENTICATIONDATABASE" ]; then + args="$args --authenticationDatabase=$OPT_AUTHENTICATIONDATABASE" + fi + if [ -n "$OPT_SSLCAFILE" ]; then + args="$args --tlsCAFile=$OPT_SSLCAFILE" + fi + if [ -n "$OPT_SSLPEMKEYFILE" ]; then + args="$args --tlsCertificateKeyFile=$OPT_SSLPEMKEYFILE" + fi + if [ "$OPT_TLS" ]; then + args="$args --tls" + fi + + echo "$args" +} + +mongo_eval() { + local script="$1" + "$MONGO_SHELL" --quiet $MONGO_CONNECT_ARGS --eval "$script" +} + +mongo_monitor_args() { + local args="" + + if [ -n "$OPT_USER" ]; then + args="$args --username=$OPT_USER" + fi + if [ -n "$OPT_PASSWORD" ]; then + args="$args --password=$OPT_PASSWORD" + fi + if [ -n "$OPT_AUTHENTICATIONDATABASE" ]; then + args="$args --authenticationDatabase=$OPT_AUTHENTICATIONDATABASE" + fi + if [ -n "$OPT_SSLCAFILE" ]; then + args="$args --sslCAFile=$OPT_SSLCAFILE" + fi + if [ -n "$OPT_SSLPEMKEYFILE" ]; then + args="$args --sslPEMKeyFile=$OPT_SSLPEMKEYFILE" + fi + if [ "$OPT_TLS" ]; then + args="$args --ssl" + fi + if [ -n "$OPT_URI" ]; then + args="$args $OPT_URI" + else + args="$args --host=$OPT_HOST --port=$OPT_PORT" + fi + + echo "$args" +} + +mongo_monitor_exec() { + local cmd="$1" + local interval="$2" + shift 2 + "$cmd" "$@" $MONGO_MONITOR_ARGS "$interval" +} + +mongo_admin_json() { + local expr="$1" + mongo_eval " + const __pt = db.getSiblingDB('admin'); + const __doc = ${expr}; + if (__doc && typeof __doc === 'object' && !Array.isArray(__doc)) { + delete __doc.ok; + delete __doc['\$clusterTime']; + delete __doc.operationTime; + } + print(${MONGO_EVAL_PREFIX}(__doc)); + " +} + +detect_mongo_topology() { + local topology + topology=$(mongo_eval " + const __pt = db.getSiblingDB('admin'); + const __serverStatus = __pt.runCommand({serverStatus: 1}) || {}; + let __clusterRole = ''; + let __hello = {}; + try { + const __cmdLineOpts = __pt.runCommand({getCmdLineOpts: 1}); + if (__cmdLineOpts && __cmdLineOpts.parsed && __cmdLineOpts.parsed.sharding && __cmdLineOpts.parsed.sharding.clusterRole) { + __clusterRole = __cmdLineOpts.parsed.sharding.clusterRole; + } + } catch (e) { + } + try { + __hello = __pt.runCommand({hello: 1}) || {}; + } catch (e) { + try { + __hello = __pt.runCommand({isMaster: 1}) || {}; + } catch (e2) { + __hello = {}; + } + } + + let __replicaSetRole = ''; + if (__hello.setName) { + if (__hello.isWritablePrimary || __hello.ismaster) { + __replicaSetRole = 'PRIMARY'; + } else if (__hello.secondary) { + __replicaSetRole = 'SECONDARY'; + } else if (__hello.arbiterOnly) { + __replicaSetRole = 'ARBITER'; + } else { + __replicaSetRole = 'MEMBER'; + } + } + + if (__clusterRole === 'shardsvr') { + __clusterRole = 'shard'; + } else if (__clusterRole === 'configsvr') { + __clusterRole = 'config-server'; + } + + print([ + __serverStatus.process || '', + __clusterRole, + __replicaSetRole, + __hello.setName || '' + ].join('\t')); + ") + + IFS=$'\t' read -r MONGO_PROCESS_TYPE MONGO_CLUSTER_ROLE MONGO_REPLICA_SET_ROLE MONGO_REPLSET_NAME < __maxLag) { + __maxLag = __lag; + } + } + print(__maxLag); + " +} + +_js_quote() { + printf '%s' "$1" | perl -0pe 's/\\/\\\\/g; s/'"'"'/\\'"'"'/g; $_="'"'"'$_'"'"'"' +} + +get_mongo_pid() { + local pid="" + if [ -n "$OPT_PORT" ]; then + pid=$(lsof -i ":${OPT_PORT}" 2>/dev/null | awk '/LISTEN/ && ($1=="mongod" || $1=="mongos") {print $2; exit}') + fi + if [ -z "$pid" ]; then + pid=$(_pidof mongod | awk '{print $1; exit}') + fi + if [ -z "$pid" ]; then + pid=$(_pidof mongos | awk '{print $1; exit}') + fi + echo "$pid" +} + +run_maybe_sudo() { + local cmd="$1" + shift + + if [ "$(id -u)" -eq 0 ]; then + "$cmd" "$@" + elif [ -n "$(_which sudo)" ] && sudo -n true >/dev/null 2>&1; then + sudo -n "$cmd" "$@" + else + "$cmd" "$@" + fi +} + +sample_ts() { + date +%F-%T | tr ':-' '_' +} + +sample_file() { + local d="$1" + local label="$2" + local ext="$3" + printf '%s/%s-%s.%s' "$d" "$(sample_ts)" "$label" "$ext" +} + +sample_file_with_ts() { + local d="$1" + local ts="$2" + local label="$3" + local ext="$4" + printf '%s/%s-%s.%s' "$d" "$ts" "$label" "$ext" +} + +state_file() { + local d="$1" + local label="$2" + printf '%s/%s' "$d" "$label" +} + +fixed_file() { + local d="$1" + local label="$2" + local ext="$3" + printf '%s/%s.%s' "$d" "$label" "$ext" +} + +sample_count() { + perl -e ' + my ($run_time, $sleep_collect) = @ARGV; + $run_time = 0 + ($run_time || 0); + $sleep_collect = 0 + ($sleep_collect || 0); + $sleep_collect = 1 if $sleep_collect <= 0; + my $count = int(($run_time / $sleep_collect) + 0.999999); + $count = 1 if $count < 1; + print $count; + ' "$1" "$2" +} + +collect_mongo_monitor_sample() { + local d="$1" + local label="$2" + local cmd="$3" + local interval="$4" + shift 4 + + local ts out err status + ts="$(sample_ts)" + out="$(sample_file_with_ts "$d" "$ts" "$label" "txt")" + err="$(sample_file_with_ts "$d" "$ts" "$label" "err")" + + mongo_monitor_exec "$cmd" "$interval" "$@" > "$out" 2>"$err" + status=$? + + if [ "$status" -ne 0 ]; then + echo "Command exited with status $status: $cmd $* $MONGO_MONITOR_ARGS $interval" >> "$err" + fi + if [ ! -s "$out" ]; then + echo "Command produced no output: $cmd $* $MONGO_MONITOR_ARGS $interval" >> "$err" + fi +} + +collect_command_sample() { + local d="$1" + local label="$2" + local cmd="$3" + shift 3 + + local ts out err status + ts="$(sample_ts)" + out="$(sample_file_with_ts "$d" "$ts" "$label" "txt")" + err="$(sample_file_with_ts "$d" "$ts" "$label" "err")" + + "$cmd" "$@" > "$out" 2>"$err" + status=$? + + if [ "$status" -ne 0 ]; then + echo "Command exited with status $status: $cmd $*" >> "$err" + fi + if [ ! -s "$out" ]; then + echo "Command produced no output: $cmd $*" >> "$err" + fi +} + +# ########################################################################### +# collection section +# ########################################################################### + +collect() { + local d="$1" + local cnt + local collect_wait + local mongo_pid="" + + cnt=$(sample_count "$OPT_RUN_TIME" "$OPT_SLEEP_COLLECT") + collect_wait="$OPT_RUN_TIME" + [ "$collect_wait" -lt 1 ] && collect_wait=1 + mongo_pid=$(get_mongo_pid) + collect_mongodb_data_one "$d" "$mongo_pid" + collect_system_data "$d" "$mongo_pid" "$cnt" + + log "Loop start: $(date +'TS %s.%N %F %T')" + collect_system_data_loop "$d" "$mongo_pid" + collect_mongodb_data_loop "$d" + date +"TS %s.%N %F %T" >> "$(state_file "$d" "heartbeat")" + + collect_mongodb_data_two "$d" + + wait_for_subshells "$collect_wait" + kill_all_subshells +} + +collect_mongodb_data_one() { + local d="$1" + local mongo_pid="$2" + + detect_mongo_topology + + : +} + +collect_mongodb_data_loop() { + local d="$1" + (mongo_admin_json "db.runCommand({serverStatus: 1})" > "$(sample_file "$d" "serverStatus" "json")") & + (mongo_admin_json "db.runCommand({currentOp: 1, active: true, idleConnections: false, allUsers: true})" > "$(sample_file "$d" "currentOp" "json")") & +} + +collect_mongodb_data_two() { + : +} + +collect_system_data() { + local d="$1" + local mongo_pid="$2" + local cnt="$3" + + if [ -f /var/log/messages ]; then + (run_maybe_sudo cp /var/log/messages "$(fixed_file "$d" "messages" "out")") 2>"$(fixed_file "$d" "messages" "err")" & + fi + + if [ -n "$CMD_VMSTAT" ]; then + (collect_command_sample "$d" "vmstat" "$CMD_VMSTAT" "$OPT_SLEEP_COLLECT" "$cnt") & + fi + if [ -n "$CMD_IOSTAT" ]; then + (collect_command_sample "$d" "iostat" "$CMD_IOSTAT" "$OPT_SLEEP_COLLECT" "$cnt") & + fi + if [ -n "$CMD_MPSTAT" ]; then + (collect_command_sample "$d" "mpstat" "$CMD_MPSTAT" -P ALL "$OPT_SLEEP_COLLECT" "$cnt") & + fi + if [ -n "$CMD_TOP" ]; then + if top -bn1 >/dev/null 2>&1; then + (collect_command_sample "$d" "top" "$CMD_TOP" -b -d "$OPT_SLEEP_COLLECT" -n "$cnt") & + elif top -l 1 >/dev/null 2>&1; then + (collect_command_sample "$d" "top" "$CMD_TOP" -l "$cnt" -s "$OPT_SLEEP_COLLECT") & + fi + fi + if [ -n "$CMD_MONGOSTAT" ]; then + (collect_mongo_monitor_sample "$d" "mongostat" "$CMD_MONGOSTAT" "$OPT_SLEEP_COLLECT" --rowcount "$cnt") & + fi + if [ -n "$CMD_MONGOTOP" ]; then + (collect_mongo_monitor_sample "$d" "mongotop" "$CMD_MONGOTOP" "$OPT_SLEEP_COLLECT" --rowcount "$cnt") & + fi +} + +collect_system_data_loop() { + local d="$1" + local mongo_pid="$2" + local disk_space_file="$PT_TMPDIR/collect-disk-space" + local mongo_proc_label="pidstat_mongod" + + disk_space "$d" > "$disk_space_file" + check_disk_space "$disk_space_file" "$OPT_DISK_BYTES_FREE" "$OPT_DISK_PCT_FREE" || return + + (ps faux > "$(sample_file "$d" "ps" "txt")") & + if [ -n "$CMD_PIDSTAT" ]; then + (collect_command_sample "$d" "pidstat_d" "$CMD_PIDSTAT" -d 1 1) & + (collect_command_sample "$d" "pidstat_u" "$CMD_PIDSTAT" -u 1 1) & + if [ -n "$mongo_pid" ]; then + [ "$MONGO_PROCESS_TYPE" = "mongos" ] && mongo_proc_label="pidstat_mongos" + (collect_command_sample "$d" "$mongo_proc_label" "$CMD_PIDSTAT" -urdwt -h -p "$mongo_pid" 1 1) & + fi + fi +} + +# ########################################################################### +# Global variables +# ########################################################################### + +TRIGGER_FUNCTION="" +RAN_WITH="" +EXIT_REASON="" +TOOL="pt-mongodb-stalk" +OKTORUN=1 +ITER=1 + +# ########################################################################### +# Plugin hooks +# ########################################################################### + +before_stalk() { :; } +before_collect() { :; } +after_collect() { :; } +after_collect_sleep() { :; } +after_interval_sleep() { :; } +after_stalk() { :; } + +# ########################################################################### +# Trigger and control flow +# ########################################################################### + +set_trg_func() { + local func="$1" + if [ -f "$func" ]; then + . "$func" + TRIGGER_FUNCTION="trg_plugin" + return 0 + fi + + func=$(echo "$func" | tr '[:upper:]' '[:lower:]') + case "$func" in + status|currentop|cpu|memory|writewait|repllag) + TRIGGER_FUNCTION="trg_$func" + return 0 + ;; + mem) + TRIGGER_FUNCTION="trg_memory" + return 0 + ;; + processcpu|process-cpu|proc_cpu|proc-cpu) + TRIGGER_FUNCTION="trg_process_cpu" + return 0 + ;; + processmemory|process-memory|proc_memory|proc-memory) + TRIGGER_FUNCTION="trg_process_memory" + return 0 + ;; + replicationlag|replciationlag|waitforreplicationlag|waitforreplciationlag) + TRIGGER_FUNCTION="trg_repllag" + return 0 + ;; + esac + + return 1 +} + +trg_status() { + local var="$1" + mongo_status_value "$var" +} + +trg_currentop() { + local var="$1" + mongo_currentop_count "$var" +} + +trg_cpu() { + local cpu user nice system idle iowait irq softirq steal guest guest_nice + local idle_all non_idle total total_delta idle_delta value + local cpu_state last_total last_idle + + read -r cpu user nice system idle iowait irq softirq steal guest guest_nice < /proc/stat || return 1 + iowait=${iowait:-0} + irq=${irq:-0} + softirq=${softirq:-0} + steal=${steal:-0} + idle_all=$((idle + iowait)) + non_idle=$((user + nice + system + irq + softirq + steal)) + total=$((idle_all + non_idle)) + cpu_state="$PT_TMPDIR/cpu-trigger-state" + + if [ ! -f "$cpu_state" ]; then + echo "$total $idle_all" > "$cpu_state" + echo 0 + return 0 + fi + + read -r last_total last_idle < "$cpu_state" || { + echo "$total $idle_all" > "$cpu_state" + echo 0 + return 0 + } + total_delta=$((total - last_total)) + idle_delta=$((idle_all - last_idle)) + echo "$total $idle_all" > "$cpu_state" + + if [ "$total_delta" -le 0 ]; then + echo 0 + return 0 + fi + + value=$(awk -v total="$total_delta" -v idle="$idle_delta" 'BEGIN { printf "%.2f\n", ((total - idle) / total) * 100 }') + echo "$value" +} + +trg_memory() { + awk ' + /^MemTotal:/ { total = $2 } + /^MemAvailable:/ { available = $2 } + /^MemFree:/ { free = $2 } + /^Buffers:/ { buffers = $2 } + /^Cached:/ { cached = $2 } + END { + if (available == "" && free >= 0) { + available = free + buffers + cached + } + if (total > 0 && available >= 0) { + printf "%.2f\n", ((total - available) / total) * 100 + } + } + ' /proc/meminfo +} + +trg_process_cpu() { + local pid value + pid=$(get_mongo_pid) + [ -n "$pid" ] || return 1 + value=$(ps -p "$pid" -o %cpu= 2>/dev/null | awk '{gsub(/,/, "."); printf "%.2f\n", $1 + 0; exit}') + [ -n "$value" ] && echo "$value" +} + +trg_process_memory() { + local pid value + pid=$(get_mongo_pid) + [ -n "$pid" ] || return 1 + value=$(ps -p "$pid" -o %mem= 2>/dev/null | awk '{gsub(/,/, "."); printf "%.2f\n", $1 + 0; exit}') + [ -n "$value" ] && echo "$value" +} + +trg_writewait() { + local value + value=$(mongo_status_value "globalLock.currentQueue.writers") + if [ -z "$value" ]; then + value=0 + fi + echo "$value" +} + +trg_repllag() { + mongo_repl_lag_seconds +} + +oktorun() { + if [ "$OKTORUN" -eq 0 ]; then + [ -z "$EXIT_REASON" ] && EXIT_REASON="OKTORUN is false" + return 1 + fi + + if [ -n "${OPT_ITERATIONS:-}" ] && [ "$OPT_ITERATIONS" -gt 0 ] && [ "$ITER" -gt "$OPT_ITERATIONS" ]; then + [ -z "$EXIT_REASON" ] && EXIT_REASON="no more iterations" + return 1 + fi + + return 0 +} + +sleep_ok() { + local seconds="$1" + local msg="${2:-""}" + if oktorun; then + [ "$msg" ] && log "$msg" + sleep "$seconds" + fi +} + +purge_samples() { + local dir + dir=$(realpath -s "$1") + local retention_time="$2" + local retention_count="$3" + + find "$dir" -maxdepth 1 -type f -mtime +"$retention_time" -regextype posix-egrep \ + -regex "$dir/[0-9]{4}(_[0-9]{2}){5}-.*" -exec rm -f '{}' \; + + if [ "$retention_count" -gt 0 ]; then + local target_count=$((retention_count + 1)) + local files_to_delete + files_to_delete=$(find "$dir" -maxdepth 1 -type f -exec basename {} \; | cut -f1 -d- | sort -r | uniq | tail -n +"${target_count}") + for prefix in $files_to_delete; do + rm -f "$dir"/"$prefix"-* 2>/dev/null + done + fi +} + +sigtrap() { + if [ "$OKTORUN" -eq 1 ]; then + warn "Caught signal, exiting" + OKTORUN=0 + else + warn "Caught signal again, forcing exit" + exit "$EXIT_STATUS" + fi +} + +stalk() { + local cycles_true=0 + local matched="" + local last_prefix="" + + while oktorun; do + local msg="" + if [ "$OPT_STALK" ]; then + local value + value=$($TRIGGER_FUNCTION "$OPT_VARIABLE") + local trg_exit_status=$? + if [ -z "$value" ]; then + warn "Detected value is empty; something failed? Trigger exit status: $trg_exit_status" + matched="" + cycles_true=0 + elif (( $(echo "$value $OPT_THRESHOLD" | awk '{print ($1 > $2)}') )); then + matched="yes" + cycles_true=$((cycles_true + 1)) + else + matched="" + cycles_true=0 + fi + + msg="Check results: $OPT_FUNCTION($OPT_VARIABLE)=$value, matched=${matched:-no}, cycles_true=$cycles_true" + if [ "$matched" ]; then + log "$msg" + else + info "$msg" + fi + elif [ "$OPT_COLLECT" ]; then + matched=1 + cycles_true=$OPT_CYCLES + msg="Not stalking; collect triggered immediately" + log "$msg" + fi + + if [ "$matched" ] && [ "$cycles_true" -ge "$OPT_CYCLES" ]; then + log "Collect $ITER triggered" + + if [ "$OPT_COLLECT" ]; then + local prefix="${OPT_PREFIX:-$(date +%F-%T | tr ':-' '_')}" + local margin="20971520" + if [ -n "$last_prefix" ]; then + margin=$(du -mc "$OPT_DEST"/"$last_prefix"-* 2>/dev/null | tail -n 1 | awk '{print $1}') + [ -n "$margin" ] || margin="20971520" + fi + + disk_space "$OPT_DEST" > "$PT_TMPDIR/$prefix-disk-space" + check_disk_space "$PT_TMPDIR/$prefix-disk-space" "$OPT_DISK_BYTES_FREE" "$OPT_DISK_PCT_FREE" "$margin" + if [ $? -eq 0 ]; then + ts "$msg" >> "$(state_file "$OPT_DEST" "trigger")" + ts "pt-mongodb-stalk ran with $RAN_WITH" >> "$(state_file "$OPT_DEST" "trigger")" + last_prefix="$prefix" + + before_collect + + local collect_start_epoch + local collect_end_epoch + local collect_elapsed + collect_start_epoch=$(date +%s) + ts "Collect $ITER started; run-time=$OPT_RUN_TIME; sleep-collect=$OPT_SLEEP_COLLECT" >> "$(state_file "$OPT_DEST" "trigger")" + + ( + collect "$OPT_DEST" + ) >> "$(state_file "$OPT_DEST" "log")" 2>&1 & + local collector_pid=$! + log "Collect $ITER PID $collector_pid" + after_collect "$collector_pid" + wait "$collector_pid" + local collector_status=$? + if [ "$collector_status" -ne 0 ]; then + warn "Collect $ITER exited with status $collector_status" + fi + collect_end_epoch=$(date +%s) + collect_elapsed=$((collect_end_epoch - collect_start_epoch)) + ts "Collect $ITER finished in ${collect_elapsed}s; sleep=$OPT_SLEEP before next iteration" >> "$(state_file "$OPT_DEST" "trigger")" + else + warn "Collect canceled because there will not be enough disk space after collecting another sample" + fi + + if [ -d "$OPT_DEST" ]; then + purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" + fi + fi + + log "Collect $ITER done" + ITER=$((ITER + 1)) + cycles_true=0 + sleep_ok "$OPT_SLEEP" "Sleeping $OPT_SLEEP seconds after collect" + after_collect_sleep + else + sleep_ok "$OPT_INTERVAL" + after_interval_sleep + fi + done + + if [ "$OPT_COLLECT" ] && [ -d "$OPT_DEST" ]; then + purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" + fi + + wait_for_subshells $((OPT_RUN_TIME * 3)) + kill_all_subshells + remove_empty_err_files "$OPT_DEST" +} + +main() { + trap sigtrap SIGHUP SIGINT SIGTERM + + RAN_WITH="--function=$OPT_FUNCTION --variable=$OPT_VARIABLE --threshold=$OPT_THRESHOLD --match=$OPT_MATCH --cycles=$OPT_CYCLES --interval=$OPT_INTERVAL --iterations=$OPT_ITERATIONS --run-time=$OPT_RUN_TIME --sleep=$OPT_SLEEP --dest=$OPT_DEST --prefix=$OPT_PREFIX --log=$OPT_LOG --pid=$OPT_PID --plugin=$OPT_PLUGIN" + + log "Starting $0 $RAN_WITH" + choose_mongo_shell + MONGO_CONNECT_ARGS="$(mongo_connect_args)" + MONGO_MONITOR_ARGS="$(mongo_monitor_args)" + + mk_tmpdir + before_stalk + stalk + after_stalk + rm_tmpdir + remove_pid_file "$OPT_PID" + + log "Exiting because ${EXIT_REASON:-completed}" + [ "$OPT_COLLECT" ] && log "Files stored in $OPT_DEST" + log "$0 exit status $EXIT_STATUS" + exit "$EXIT_STATUS" +} + +if [ "${0##*/}" = "$TOOL" ] || [ "${0##*/}" = "bash" -a "${_:-""}" = "$0" ]; then + mk_tmpdir + parse_options "$0" "${@:-""}" + usage_or_errors "$0" + po_status=$? + rm_tmpdir + if [ "$po_status" -ne 0 ]; then + [ "$OPT_ERRS" -gt 0 ] && exit 1 + exit 0 + fi + + if ! set_trg_func "$OPT_FUNCTION"; then + option_error "Invalid --function value: $OPT_FUNCTION" + usage_or_errors "$0" >/dev/null || exit 1 + fi + + if [ "$OPT_PLUGIN" ]; then + if [ -f "$OPT_PLUGIN" ]; then + . "$OPT_PLUGIN" + else + die "Invalid --plugin value: $OPT_PLUGIN is not a file" + fi + fi + + if [ -n "$OPT_ASK_PASS" ]; then + stty_orig=$(stty -g) + echo -n "Enter MongoDB password: " + stty -echo + read OPT_PASSWORD + stty "$stty_orig" + echo + fi + + if [ -z "$OPT_STALK" ] && [ "$OPT_COLLECT" ]; then + OPT_CYCLES=0 + fi + + choose_mongo_shell + MONGO_CONNECT_ARGS="$(mongo_connect_args)" + MONGO_MONITOR_ARGS="$(mongo_monitor_args)" + + mongo_eval "db.getSiblingDB('admin').runCommand({ping: 1})" >/dev/null 2>&1 \ + || die "Cannot connect to MongoDB. Check that MongoDB is running and that the options are correct." + + if [ "$OPT_COLLECT" ]; then + [ -d "$OPT_DEST" ] || mkdir -p "$OPT_DEST" || die "Cannot make --dest $OPT_DEST" + ( + set -e + touch "$OPT_DEST/test" + rm "$OPT_DEST/test" + ) || die "Cannot read and write files to --dest $OPT_DEST" + fi + + if [ "$OPT_STALK" ] && [ "$OPT_DAEMONIZE" ]; then + touch "$OPT_LOG" || die "Cannot write to --log $OPT_LOG" + make_pid_file "$OPT_PID" $$ + main "${@:-""}" >"$OPT_LOG" 2>&1 & + echo "$!" > "$OPT_PID" + else + [ "$OPT_STALK" ] && make_pid_file "$OPT_PID" $$ + main "${@:-""}" + fi +fi + +:<<'DOCUMENTATION' +=pod + +=head1 NAME + +pt-mongodb-stalk - Collect forensic data about MongoDB when problems occur. + +=head1 SYNOPSIS + +Usage: pt-mongodb-stalk [OPTIONS] + +=head1 DESCRIPTION + +pt-mongodb-stalk watches a MongoDB server for a trigger condition and collects +diagnostic data when that trigger occurs. It follows the same basic operating +model as C, but uses MongoDB administration commands instead of +MySQL commands. + +The default trigger watches C. You can also +watch C, host CPU usage, host memory usage, queued writers, and +replica set replication lag. + +=head1 OPTIONS + +=over + +=item --ask-pass + +Prompt for a password when connecting to MongoDB. + +=item --authenticationDatabase + +type: string; default: admin + +Authentication database for the MongoDB shell connection. + +=item --collect + +default: yes; negatable: yes + +Collect diagnostic data when the trigger occurs. + +=item --config + +type: string + +Read this comma-separated list of config files. If specified, this must be the +first option on the command line. + +=item --cycles + +type: int; default: 5 + +How many times L<"--variable"> must be greater than L<"--threshold"> before +triggering L<"--collect">. + +=item --daemonize + +Daemonize the tool and log output to L<"--log">. + +=item --dest + +type: string; default: /var/lib/pt-mongodb-stalk + +Where to save diagnostic data. + +=item --disk-bytes-free + +type: size; default: 100M + +Do not collect if the disk has less than this much free space. + +=item --disk-pct-free + +type: int; default: 5 + +Do not collect if the disk has less than this percent free space. + +=item --function + +type: string; default: status + +Trigger source. Valid built-in values are C, C, C, +C, C, and C. + +With C, L<"--variable"> is a dot-separated path inside +C. Example: + + --function status --variable connections.current --threshold 200 + +With C, L<"--variable"> is a dot-separated field path inside each +C document and L<"--match"> is a regex. The trigger value is +the number of matching operations. Example: + + --function currentop --variable command.aggregate --match '^orders$' --threshold 10 + +With C, the trigger value is host CPU busy percentage from C. +With C, the trigger value is host memory used percentage from +C. C is accepted as an alias for C. Example: + + --function cpu --threshold 85 + --function memory --threshold 90 + +With C, the trigger value is +C. Example: + + --function writewait --threshold 5 + +With C, the trigger value is the maximum replica set member lag behind +the primary, in seconds, from C. C and +C are accepted as aliases. Example: + + --function repllag --threshold 30 + +You can also specify a file that defines C. + +=item --help + +Print help and exit. + +=item --host + +short form: -h; type: string; default: localhost + +Host to connect to. + +=item --interval + +type: int; default: 1 + +How often to check the trigger, in seconds. + +=item --iterations + +type: int + +How many collections to perform before exiting. + +=item --log + +type: string; default: /var/log/pt-mongodb-stalk.log + +Print all output to this file when daemonized. + +=item --match + +type: string + +Regex pattern used with L<"--function"> C. + +=item --password + +short form: -p; type: string + +Password to use when connecting. + +=item --pid + +type: string; default: /var/run/pt-mongodb-stalk.pid + +Create the given PID file. + +=item --plugin + +type: string + +Load a plugin that defines any of the standard C or C +hooks. + +=item --port + +short form: -P; type: int; default: 27017 + +Port number to use for connection. + +=item --prefix + +type: string + +Filename prefix for diagnostic samples. + +=item --retention-count + +type: int; default: 0 + +Keep data for the last N runs. + +=item --retention-time + +type: int; default: 30 + +Number of days to retain collected samples. + +=item --run-time + +type: int; default: 30 + +How long interval collectors should run when the trigger occurs. + +=item --sleep + +type: int; default: 300 + +How long to sleep after collecting. + +=item --sleep-collect + +type: int; default: 1 + +Polling interval for interval collectors, in seconds. + +=item --stalk + +default: yes; negatable: yes + +Watch the server and wait for the trigger to occur. Specify C<--no-stalk> to +collect immediately. + +=item --threshold + +type: float; default: 100 + +Collection is triggered when L<"--variable"> is greater than this value. + +=item --tls + +default: ; negatable: yes + +Enable TLS for the MongoDB shell connection. + +=item --sslCAFile + +type: string + +Path to the TLS CA file. + +=item --sslPEMKeyFile + +type: string + +Path to the TLS client certificate and key file. + +=item --uri + +type: string + +Full MongoDB URI to connect with. + +=item --user + +short form: -u; type: string + +User for login. + +=item --variable + +type: string; default: connections.current + +Variable to watch inside C or C. This option is ignored +by C, C, C, and C. + +=item --verbose + +type: int; default: 3 + +Print level of information. Values: 1 errors, 2 matching triggers and +collection info, 3 non-matching triggers. + +=item --version + +Print version and exit. + +=back + +=head1 EXAMPLES + +Run in stalking mode and collect twice when the trigger is met: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function status --variable connections.current --threshold 50 --cycles 3 --interval 1 --iterations 2 \ + --dest /tmp/pt-mongodb-stalk + +Run immediately without stalking and collect one short sample: + + pt-mongodb-stalk \ + --host localhost --port 30004 --user admin --password admin --authenticationDatabase admin \ + --no-stalk --iterations 1 --run-time 6 --sleep-collect 1 \ + --dest /tmp/pt-mongodb-stalk + +Run immediately without stalking and collect multiple short runs: + + pt-mongodb-stalk \ + --host localhost --port 30000 --user admin --password admin --authenticationDatabase admin \ + --no-stalk --iterations 3 --run-time 6 --sleep-collect 1 --sleep 1 \ + --dest /tmp/pt-mongodb-stalk + +Run immediately without stalking and collect fewer, more widely spaced samples: + + pt-mongodb-stalk \ + --host localhost --port 27000 --user admin --password admin --authenticationDatabase admin \ + --no-stalk --iterations 1 --run-time 10 --sleep-collect 2 \ + --dest /tmp/pt-mongodb-stalk + +Run in stalking mode using currentOp matches instead of a serverStatus metric: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function currentop --variable command.aggregate --match '^orders$' --threshold 10 --cycles 2 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk + +Run in stalking mode when host CPU is above 85 percent: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function cpu --threshold 85 --cycles 3 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk + +Run in stalking mode when host memory is above 90 percent: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function memory --threshold 90 --cycles 3 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk + +Run in stalking mode when replica set lag is above 30 seconds: + + pt-mongodb-stalk \ + --host localhost --port 30001 --user admin --password admin --authenticationDatabase admin \ + --function repllag --threshold 30 --cycles 2 --interval 1 --iterations 1 \ + --dest /tmp/pt-mongodb-stalk + +=head1 OUTPUT + +When the trigger condition is met for the configured number of consecutive +cycles, the tool collects into L<"--dest">. Snapshot commands run once per +collection iteration and are stored as timestamped files. For example: + + 2026_04_24_10_00_01-serverStatus.json + 2026_04_24_10_00_01-currentOp.json + 2026_04_24_10_00_01-ps.txt + +Interval commands run once per collection iteration using L<"--sleep-collect"> +as their polling interval and a count derived from L<"--run-time">. For +example, C<--run-time 5 --sleep-collect 1> runs commands like C +and stores the result in one timestamped file. + +The collection window is capped by L<"--run-time">. After a collection +finishes, the tool waits L<"--sleep"> seconds before the next trigger check or +collection iteration. Collections do not overlap. + +The collector also writes these fixed files in the destination directory: + + heartbeat + log + trigger + +=head1 COLLECTED DATA + +MongoDB data collected once per collection iteration: + + serverStatus + currentOp + +MongoDB interval tools collected once per collection iteration: + + mongostat + mongotop + +MongoDB JSON output from shell commands is cleaned before writing: C, +C<$clusterTime>, and C are removed. + +System data collected once per collection iteration, depending on tool +availability: + + ps faux + pidstat -d + pidstat -u + pidstat -urdwt for mongod or mongos + +System interval tools collected once per collection iteration, depending on +tool availability: + + vmstat + iostat + mpstat + top + +The process-specific C file is named C or +C according to the detected MongoDB process type. Topology +is detected internally for this purpose, but no topology summary file is +written. + +If C exists, the tool also copies it to C. + +=head1 OUTPUT CLEANUP + +At the end of a run, zero-byte C<.err> files are removed and non-empty C<.err> +files are kept. The collector enforces disk-space safety checks, but uses +internal temporary files instead of writing disk-space snapshots into +L<"--dest">. + +=head1 NOTES + +This tool is intended for Linux systems. A separate summary-style tool should +collect broader one-time server and MongoDB metadata; C is +focused on runtime sampling around a trigger event. + +=head1 COPYRIGHT, LICENSE, AND WARRANTY + +This program is copyright 2011-2026 Percona LLC and/or its affiliates. + +THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + +=cut +DOCUMENTATION diff --git a/t/pt-mongodb-stalk/pt-mongodb-stalk.t b/t/pt-mongodb-stalk/pt-mongodb-stalk.t new file mode 100644 index 000000000..f1a714a7d --- /dev/null +++ b/t/pt-mongodb-stalk/pt-mongodb-stalk.t @@ -0,0 +1,181 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use File::Path qw(make_path); +use File::Temp qw(tempdir); +use Test::More; + +use PerconaTest; + +my $tool = "$trunk/bin/pt-mongodb-stalk"; +my $tmpdir = tempdir("pt-mongodb-stalk.XXXXXX", TMPDIR => 1, CLEANUP => 1); +my $fakebin = "$tmpdir/bin"; +my $run_no = 0; + +make_path($fakebin); + +sub write_fake_cmd { + my ($name, $body) = @_; + my $file = "$fakebin/$name"; + open my $fh, ">", $file or die "Cannot write $file: $OS_ERROR"; + print {$fh} $body; + close $fh; + chmod 0755, $file or die "Cannot chmod $file: $OS_ERROR"; + return $file; +} + +sub run_capture { + my (@cmd) = @_; + $run_no++; + my $out = "$tmpdir/run-$run_no.out"; + my $pid = fork(); + die "Cannot fork: $OS_ERROR" unless defined $pid; + + if ( !$pid ) { + open STDOUT, ">", $out or die "Cannot redirect stdout: $OS_ERROR"; + open STDERR, ">&", \*STDOUT or die "Cannot redirect stderr: $OS_ERROR"; + exec @cmd; + die "Cannot exec $cmd[0]: $OS_ERROR"; + } + + waitpid($pid, 0); + my $status = $CHILD_ERROR >> 8; + my $text = slurp_file($out); + return ($status, $text); +} + +sub glob_count { + my ($pattern) = @_; + my @files = glob($pattern); + return scalar @files; +} + +write_fake_cmd("mongosh", <<'SH'); +#!/usr/bin/env bash +[ -n "${MONGOSH_LOG:-}" ] && printf '%s\n' "$*" >> "$MONGOSH_LOG" +case "$*" in + *"__serverStatus.process"*) + printf 'mongod\tshard\tPRIMARY\trs0\n' + ;; + *"currentOp: 1"*) + printf '{"inprog":[]}\n' + ;; + *"serverStatus: 1"*) + printf '{"process":"mongod","connections":{"current":5},"globalLock":{"currentQueue":{"writers":0}}}\n' + ;; + *"replSetGetStatus: 1"*) + printf '{"members":[]}\n' + ;; +esac +exit 0 +SH + +write_fake_cmd("mongostat", <<'SH'); +#!/usr/bin/env bash +echo "insert query update delete getmore command dirty used flushes vsize res qrw arw net_in net_out conn time" +echo "0 0 0 0 0 1|0 0.0% 1.0% 0 100M 50M 0|0 0|0 1k 2k 1 00:00:00" +echo "2026-05-14T00:00:00.000+0000 connected to: mongodb://localhost:27017/" >&2 +SH + +write_fake_cmd("mongotop", <<'SH'); +#!/usr/bin/env bash +echo "ns total read write" +echo "admin.system.version 1ms 1ms 0ms" +echo "2026-05-14T00:00:00.000+0000 connected to: mongodb://localhost:27017/" >&2 +SH + +write_fake_cmd("df", <<'SH'); +#!/usr/bin/env bash +cat </dev/null`); + +ok(-f "$dest/trigger", "creates trigger file"); +ok(-f "$dest/heartbeat", "creates heartbeat file"); +ok(-f "$dest/log", "creates log file"); + +is(glob_count("$dest/*-serverStatus.json"), 1, "collects serverStatus once"); +is(glob_count("$dest/*-currentOp.json"), 1, "collects currentOp once"); +is(glob_count("$dest/*-mongostat.txt"), 1, "collects mongostat once"); +is(glob_count("$dest/*-mongotop.txt"), 1, "collects mongotop once"); +is(glob_count("$dest/*-ps.txt"), 1, "collects ps once"); +is(glob_count("$dest/*-pidstat_mongod.txt"), 1, "collects process pidstat with mongod label"); + +my @mongotop_err = glob("$dest/*-mongotop.err"); +is(scalar @mongotop_err, 1, "keeps non-empty mongotop stderr"); +like(slurp_file($mongotop_err[0]), qr/connected to: mongodb:\/\/localhost:27017/, "mongotop stderr keeps connection banner"); + +my $mongosh_log = slurp_file($ENV{MONGOSH_LOG}); +like( + $mongosh_log, + qr/currentOp: 1, active: true, idleConnections: false, allUsers: true/, + "currentOp command includes allUsers" +); + +ok(!-e "$dest/topology", "does not write topology file"); +ok(!-e "$dest/disk-space", "does not write disk-space file"); + +done_testing;