From 87b98ec975bfa5fd545a6268fcbec4c24a1a7c93 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 24 Feb 2026 15:23:50 -0800 Subject: [PATCH 1/3] feature: flamegraph flag to customize java profiling data collection Signed-off-by: Harper, Jason M --- cmd/flamegraph/flamegraph.go | 40 +++++-- cmd/flamegraph/flamegraph_renderers.go | 23 +++- cmd/flamegraph/flamegraph_tables.go | 20 +++- internal/script/scripts.go | 139 +++++++++++++++---------- 4 files changed, 159 insertions(+), 63 deletions(-) diff --git a/cmd/flamegraph/flamegraph.go b/cmd/flamegraph/flamegraph.go index cf9c9130..5c25c5bd 100644 --- a/cmd/flamegraph/flamegraph.go +++ b/cmd/flamegraph/flamegraph.go @@ -52,6 +52,8 @@ var ( flagNoSystemSummary bool flagMaxDepth int flagPerfEvent string + flagSampleTypes []string + flagAsprofArguments string ) const ( @@ -61,8 +63,17 @@ const ( flagNoSystemSummaryName = "no-summary" flagMaxDepthName = "max-depth" flagPerfEventName = "perf-event" + flagSampleTypesName = "sample" + flagAsprofArgumentsName = "asprof-args" ) +const ( + SampleTypeNative = "native" + SampleTypeJava = "java" +) + +var SampleTypeOptions = []string{SampleTypeNative, SampleTypeJava} + func init() { Cmd.Flags().StringVar(&flagInput, app.FlagInputName, "", "") Cmd.Flags().StringSliceVar(&flagFormat, app.FlagFormatName, []string{report.FormatHtml}, "") @@ -72,7 +83,8 @@ func init() { Cmd.Flags().BoolVar(&flagNoSystemSummary, flagNoSystemSummaryName, false, "") Cmd.Flags().IntVar(&flagMaxDepth, flagMaxDepthName, 0, "") Cmd.Flags().StringVar(&flagPerfEvent, flagPerfEventName, "cycles:P", "") - + Cmd.Flags().StringSliceVar(&flagSampleTypes, flagSampleTypesName, []string{SampleTypeNative, SampleTypeJava}, "") + Cmd.Flags().StringVar(&flagAsprofArguments, flagAsprofArgumentsName, "-t -F probesp+vtable", "") workflow.AddTargetFlags(Cmd) Cmd.SetUsageFunc(usageFunc) @@ -106,6 +118,10 @@ func usageFunc(cmd *cobra.Command) error { func getFlagGroups() []app.FlagGroup { var groups []app.FlagGroup flags := []app.Flag{ + { + Name: flagSampleTypesName, + Help: fmt.Sprintf("choose sample type(s) from: %s", strings.Join(SampleTypeOptions, ", ")), + }, { Name: flagDurationName, Help: "number of seconds to run the collection. If 0, the collection will run indefinitely. Ctrl+c to stop.", @@ -122,6 +138,10 @@ func getFlagGroups() []app.FlagGroup { Name: flagPerfEventName, Help: "perf event to use for native sampling (e.g., cpu-cycles, instructions, cache-misses, branches, context-switches, mem-loads, mem-stores, etc.)", }, + { + Name: flagAsprofArgumentsName, + Help: "arguments to pass to async-profiler, e.g., $ asprof start -i .", + }, { Name: flagMaxDepthName, Help: "maximum render depth of call stack in flamegraph (0 = no limit)", @@ -181,6 +201,12 @@ func validateFlags(cmd *cobra.Command, args []string) error { if flagMaxDepth < 0 { return workflow.FlagValidationError(cmd, "max depth must be 0 or greater") } + // validate sample types + for _, sampleType := range flagSampleTypes { + if !slices.Contains(SampleTypeOptions, sampleType) { + return workflow.FlagValidationError(cmd, fmt.Sprintf("sample type options are: %s", strings.Join(SampleTypeOptions, ", "))) + } + } // common target flags if err := workflow.ValidateTargetFlags(cmd); err != nil { return workflow.FlagValidationError(cmd, err.Error()) @@ -198,11 +224,13 @@ func runCmd(cmd *cobra.Command, args []string) error { Cmd: cmd, ReportNamePost: "flame", ScriptParams: map[string]string{ - "Frequency": strconv.Itoa(flagFrequency), - "Duration": strconv.Itoa(flagDuration), - "PIDs": strings.Join(util.IntSliceToStringSlice(flagPids), ","), - "MaxDepth": strconv.Itoa(flagMaxDepth), - "PerfEvent": flagPerfEvent, + "Frequency": strconv.Itoa(flagFrequency), + "Duration": strconv.Itoa(flagDuration), + "PIDs": strings.Join(util.IntSliceToStringSlice(flagPids), ","), + "MaxDepth": strconv.Itoa(flagMaxDepth), + "PerfEvent": flagPerfEvent, + "SampleTypes": strings.Join(flagSampleTypes, ","), + "AsprofArguments": flagAsprofArguments, }, Tables: tables, Input: flagInput, diff --git a/cmd/flamegraph/flamegraph_renderers.go b/cmd/flamegraph/flamegraph_renderers.go index 5cd2c7fb..e81e3ca0 100644 --- a/cmd/flamegraph/flamegraph_renderers.go +++ b/cmd/flamegraph/flamegraph_renderers.go @@ -229,8 +229,27 @@ func callStackFrequencyTableHTMLRenderer(tableValues table.TableValues, targetNa slog.Error("didn't find expected field (Perf Event) in table", slog.String("error", err.Error())) return out } + if len(tableValues.Fields[perfEventFieldIndex].Values) == 0 { + slog.Error("no values for perf event field in table") + return out + } perfEvent := tableValues.Fields[perfEventFieldIndex].Values[0] - out += renderFlameGraph(fmt.Sprintf("Native (%s)", perfEvent), tableValues, "Native Stacks") - out += renderFlameGraph("Java (async-profiler)", tableValues, "Java Stacks") + out += renderFlameGraph(fmt.Sprintf("Native (perf record -e %s)", perfEvent), tableValues, "Native Stacks") + + // get the asprof arguments from the table values + asprofArgumentsFieldIndex, err := table.GetFieldIndex("Asprof Arguments", tableValues) + if err != nil { + slog.Error("didn't find expected field (Asprof Arguments) in table", slog.String("error", err.Error())) + return out + } + if len(tableValues.Fields[asprofArgumentsFieldIndex].Values) == 0 { + slog.Error("no values for asprof arguments field in table") + return out + } + asprofArguments := tableValues.Fields[asprofArgumentsFieldIndex].Values[0] + if asprofArguments != "" { + asprofArguments = " " + asprofArguments + } + out += renderFlameGraph(fmt.Sprintf("Java (asprof start%s)", asprofArguments), tableValues, "Java Stacks") return out } diff --git a/cmd/flamegraph/flamegraph_tables.go b/cmd/flamegraph/flamegraph_tables.go index 60b667da..c3efb9bc 100644 --- a/cmd/flamegraph/flamegraph_tables.go +++ b/cmd/flamegraph/flamegraph_tables.go @@ -38,6 +38,7 @@ func flameGraphTableValues(outputs map[string]script.ScriptOutput) []table.Field {Name: "Java Stacks", Values: []string{javaFoldedFromOutput(outputs)}}, {Name: "Maximum Render Depth", Values: []string{maxRenderDepthFromOutput(outputs)}}, {Name: "Perf Event", Values: []string{perfEventFromOutput(outputs)}}, + {Name: "Asprof Arguments", Values: []string{asprofArgumentsFromOutput(outputs)}}, } return fields } @@ -104,7 +105,6 @@ func nativeFoldedFromOutput(outputs map[string]script.ScriptOutput) string { } } if dwarfFolded == "" && fpFolded == "" { - slog.Warn("no native folded stacks found") // "event syntax error: 'foo'" indicates that the perf event specified is invalid/unsupported if strings.Contains(outputs[script.FlameGraphScriptName].Stderr, "event syntax error") { slog.Error("unsupported perf event specified", slog.String("error", outputs[script.FlameGraphScriptName].Stderr)) @@ -154,6 +154,24 @@ func perfEventFromOutput(outputs map[string]script.ScriptOutput) string { return "" } +func asprofArgumentsFromOutput(outputs map[string]script.ScriptOutput) string { + if outputs[script.FlameGraphScriptName].Stdout == "" { + slog.Warn("collapsed call stack output is empty") + return "" + } + sections := extract.GetSectionsFromOutput(outputs[script.FlameGraphScriptName].Stdout) + if len(sections) == 0 { + slog.Warn("no sections in collapsed call stack output") + return "" + } + for header, content := range sections { + if header == "asprof_arguments" { + return strings.TrimSpace(content) + } + } + return "" +} + // ProcessStacks ... // [processName][callStack]=count type ProcessStacks map[string]Stacks diff --git a/internal/script/scripts.go b/internal/script/scripts.go index 3e3c689f..520d7d8d 100644 --- a/internal/script/scripts.go +++ b/internal/script/scripts.go @@ -1633,43 +1633,67 @@ duration={{.Duration}} frequency={{.Frequency}} maxdepth={{.MaxDepth}} perf_event={{.PerfEvent}} +sample_types={{.SampleTypes}} +read -r -a asprof_arguments <<< "{{.AsprofArguments}}" + +# determine which sample types to collect based on the input parameter +sample_native=0 +sample_java=0 +IFS=',' read -r -a sample_type_array <<< "$sample_types" +for st in "${sample_type_array[@]}"; do + if [[ "$st" == "native" ]]; then + sample_native=1 + elif [[ "$st" == "java" ]]; then + sample_java=1 + fi +done ap_interval=0 if [ "$frequency" -ne 0 ]; then ap_interval=$((1000000000 / frequency)) fi +# Restore original settings +restore_settings() { + echo "$perf_event_paranoid" > /proc/sys/kernel/perf_event_paranoid + echo "$kptr_restrict" > /proc/sys/kernel/kptr_restrict +} + # Function to stop profiling stop_profiling() { - if [ -n "$perf_fp_pid" ]; then - kill -0 "$perf_fp_pid" 2>/dev/null && kill -INT "$perf_fp_pid" - wait "$perf_fp_pid" || true + if [ $sample_native -eq 1 ]; then + if [ -n "$perf_fp_pid" ]; then + kill -0 "$perf_fp_pid" 2>/dev/null && kill -INT "$perf_fp_pid" + wait "$perf_fp_pid" || true + fi + if [ -n "$perf_dwarf_pid" ]; then + kill -0 "$perf_dwarf_pid" 2>/dev/null && kill -INT "$perf_dwarf_pid" + wait "$perf_dwarf_pid" || true + fi fi - if [ -n "$perf_dwarf_pid" ]; then - kill -0 "$perf_dwarf_pid" 2>/dev/null && kill -INT "$perf_dwarf_pid" - wait "$perf_dwarf_pid" || true + if [ $sample_java -eq 1 ]; then + for pid in "${java_pids[@]}"; do + async-profiler/bin/asprof stop -o collapsed -f ap_folded_"$pid" "$pid" + done fi - for pid in "${java_pids[@]}"; do - async-profiler/bin/asprof stop -o collapsed -f ap_folded_"$pid" "$pid" - done - # Restore original settings - echo "$perf_event_paranoid" > /proc/sys/kernel/perf_event_paranoid - echo "$kptr_restrict" > /proc/sys/kernel/kptr_restrict + restore_settings } # Function to collapse perf data collapse_perf_data() { - if [ -f perf_dwarf_data ]; then - perf script -i perf_dwarf_data > perf_dwarf_stacks - stackcollapse-perf perf_dwarf_stacks > perf_dwarf_folded - else - echo "Error: perf_dwarf_data file not found" >&2 - fi - if [ -f perf_fp_data ]; then - perf script -i perf_fp_data > perf_fp_stacks - stackcollapse-perf perf_fp_stacks > perf_fp_folded - else - echo "Error: perf_fp_data file not found" >&2 + if [ $sample_native -eq 1 ]; then + if [ -f perf_dwarf_data ]; then + perf script -i perf_dwarf_data > perf_dwarf_stacks + stackcollapse-perf perf_dwarf_stacks > perf_dwarf_folded + else + echo "Error: perf_dwarf_data file not found" >&2 + fi + if [ -f perf_fp_data ]; then + perf script -i perf_fp_data > perf_fp_stacks + stackcollapse-perf perf_fp_stacks > perf_fp_folded + else + echo "Error: perf_fp_data file not found" >&2 + fi fi } @@ -1678,8 +1702,11 @@ print_results() { echo "########## maximum depth ##########" echo "$maxdepth" - echo "########## perf_event ##########" - echo "$perf_event" + echo "########## perf_event ##########" + echo "$perf_event" + + echo "########## asprof_arguments ##########" + printf '%s\n' "${asprof_arguments[@]}" if [ -f perf_dwarf_folded ]; then echo "########## perf_dwarf ##########" @@ -1733,7 +1760,7 @@ if [ -n "$pids" ]; then fi else echo "Error: Process $p is not running." >&2 - stop_profiling + restore_settings exit 1 fi done @@ -1741,37 +1768,41 @@ else mapfile -t java_pids < <(pgrep java) fi +if [ $sample_native -eq 1 ]; then # Start profiling with perf in frame pointer mode -if [ -n "$pids" ]; then - perf record -e "$perf_event" -F "$frequency" -p "$pids" -g -o perf_fp_data -m 129 & -else - perf record -e "$perf_event" -F "$frequency" -a -g -o perf_fp_data -m 129 & -fi -perf_fp_pid=$! -if ! kill -0 $perf_fp_pid 2>/dev/null; then - echo "Failed to start perf record in frame pointer mode" >&2 - stop_profiling - exit 1 -fi + if [ -n "$pids" ]; then + perf record -e "$perf_event" -F "$frequency" -p "$pids" -g -o perf_fp_data -m 129 & + else + perf record -e "$perf_event" -F "$frequency" -a -g -o perf_fp_data -m 129 & + fi + perf_fp_pid=$! + if ! kill -0 $perf_fp_pid 2>/dev/null; then + echo "Failed to start perf record in frame pointer mode" >&2 + stop_profiling + exit 1 + fi -# Start profiling with perf in dwarf mode -if [ -n "$pids" ]; then - perf record -e "$perf_event" -F "$frequency" -p "$pids" -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & -else - perf record -e "$perf_event" -F "$frequency" -a -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & -fi -perf_dwarf_pid=$! -if ! kill -0 $perf_dwarf_pid 2>/dev/null; then - echo "Failed to start perf record in dwarf mode" >&2 - stop_profiling - exit 1 -fi + # Start profiling with perf in dwarf mode + if [ -n "$pids" ]; then + perf record -e "$perf_event" -F "$frequency" -p "$pids" -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & + else + perf record -e "$perf_event" -F "$frequency" -a -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & + fi + perf_dwarf_pid=$! + if ! kill -0 $perf_dwarf_pid 2>/dev/null; then + echo "Failed to start perf record in dwarf mode" >&2 + stop_profiling + exit 1 + fi +fi # if sample_native -# Start profiling Java with async-profiler for each Java PID -for pid in "${java_pids[@]}"; do - java_cmds+=("$(tr '\000' ' ' < /proc/"$pid"/cmdline)") - async-profiler/bin/asprof start -i "$ap_interval" -F probesp+vtable "$pid" -done +if [ $sample_java -eq 1 ]; then + # Start profiling Java with async-profiler for each Java PID + for pid in "${java_pids[@]}"; do + java_cmds+=("$(tr '\000' ' ' < /proc/"$pid"/cmdline)") + async-profiler/bin/asprof start "${asprof_arguments[@]}" -i "$ap_interval" "$pid" + done +fi # profiling has been started, set up trap to finalize on interrupt trap finalize INT TERM EXIT From 6c1697e9f5e8ad1dcc2f8e5f6c3682c9f51e1937 Mon Sep 17 00:00:00 2001 From: Jason Harper <78619061+harp-intel@users.noreply.github.com> Date: Tue, 24 Feb 2026 15:45:04 -0800 Subject: [PATCH 2/3] issue warning when java requested, but not java pids found Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- internal/script/scripts.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/script/scripts.go b/internal/script/scripts.go index 520d7d8d..356e4c83 100644 --- a/internal/script/scripts.go +++ b/internal/script/scripts.go @@ -1797,11 +1797,15 @@ if [ $sample_native -eq 1 ]; then fi # if sample_native if [ $sample_java -eq 1 ]; then - # Start profiling Java with async-profiler for each Java PID - for pid in "${java_pids[@]}"; do - java_cmds+=("$(tr '\000' ' ' < /proc/"$pid"/cmdline)") - async-profiler/bin/asprof start "${asprof_arguments[@]}" -i "$ap_interval" "$pid" - done + if [ ${#java_pids[@]} -eq 0 ]; then + echo "Warning: Java sampling was requested, but no Java processes were found; skipping Java profiling." >&2 + else + # Start profiling Java with async-profiler for each Java PID + for pid in "${java_pids[@]}"; do + java_cmds+=("$(tr '\000' ' ' < /proc/"$pid"/cmdline)") + async-profiler/bin/asprof start "${asprof_arguments[@]}" -i "$ap_interval" "$pid" + done + fi fi # profiling has been started, set up trap to finalize on interrupt From 0dc576fac18d9fed7bc8427688e8f03d85f737ee Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 24 Feb 2026 15:50:34 -0800 Subject: [PATCH 3/3] address review suggestions Signed-off-by: Harper, Jason M --- cmd/flamegraph/flamegraph.go | 3 +++ internal/script/scripts.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/cmd/flamegraph/flamegraph.go b/cmd/flamegraph/flamegraph.go index 5c25c5bd..2e9d109e 100644 --- a/cmd/flamegraph/flamegraph.go +++ b/cmd/flamegraph/flamegraph.go @@ -207,6 +207,9 @@ func validateFlags(cmd *cobra.Command, args []string) error { return workflow.FlagValidationError(cmd, fmt.Sprintf("sample type options are: %s", strings.Join(SampleTypeOptions, ", "))) } } + if len(flagSampleTypes) == 0 { + return workflow.FlagValidationError(cmd, "at least one sample type must be specified") + } // common target flags if err := workflow.ValidateTargetFlags(cmd); err != nil { return workflow.FlagValidationError(cmd, err.Error()) diff --git a/internal/script/scripts.go b/internal/script/scripts.go index 356e4c83..7e4e9073 100644 --- a/internal/script/scripts.go +++ b/internal/script/scripts.go @@ -1804,6 +1804,12 @@ if [ $sample_java -eq 1 ]; then for pid in "${java_pids[@]}"; do java_cmds+=("$(tr '\000' ' ' < /proc/"$pid"/cmdline)") async-profiler/bin/asprof start "${asprof_arguments[@]}" -i "$ap_interval" "$pid" + asprof_pid=$! + if ! kill -0 $asprof_pid 2>/dev/null; then + echo "Failed to start async-profiler for PID $pid" >&2 + stop_profiling + exit 1 + fi done fi fi