Skip to content

feature: flamegraph flag to customize java profiling data collection #649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
harp-intel wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feature: flamegraph flag to customize java profiling data collection #649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
87b98ec
feature: flamegraph flag to customize java profiling data collection
harp-intel Feb 24, 2026
6c1697e
issue warning when java requested, but not java pids found
harp-intel Feb 24, 2026
0dc576f
address review suggestions
harp-intel Feb 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 37 additions & 6 deletions cmd/flamegraph/flamegraph.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ var (
flagNoSystemSummary bool
flagMaxDepth int
flagPerfEvent string
flagSampleTypes []string
flagAsprofArguments string
)

const (
Expand All @@ -61,8 +63,17 @@ const (
flagNoSystemSummaryName = "no-summary"
flagMaxDepthName = "max-depth"
flagPerfEventName = "perf-event"
flagSampleTypesName = "sample"
flagAsprofArgumentsName = "asprof-args"
)

const (
SampleTypeNative = "native"
SampleTypeJava = "java"
)

var SampleTypeOptions = []string{SampleTypeNative, SampleTypeJava}

func init() {
Cmd.Flags().StringVar(&flagInput, app.FlagInputName, "", "")
Cmd.Flags().StringSliceVar(&flagFormat, app.FlagFormatName, []string{report.FormatHtml}, "")
Expand All @@ -72,7 +83,8 @@ func init() {
Cmd.Flags().BoolVar(&flagNoSystemSummary, flagNoSystemSummaryName, false, "")
Cmd.Flags().IntVar(&flagMaxDepth, flagMaxDepthName, 0, "")
Cmd.Flags().StringVar(&flagPerfEvent, flagPerfEventName, "cycles:P", "")

Cmd.Flags().StringSliceVar(&flagSampleTypes, flagSampleTypesName, []string{SampleTypeNative, SampleTypeJava}, "")
Cmd.Flags().StringVar(&flagAsprofArguments, flagAsprofArgumentsName, "-t -F probesp+vtable", "")
workflow.AddTargetFlags(Cmd)

Cmd.SetUsageFunc(usageFunc)
Expand Down Expand Up @@ -106,6 +118,10 @@ func usageFunc(cmd *cobra.Command) error {
func getFlagGroups() []app.FlagGroup {
var groups []app.FlagGroup
flags := []app.Flag{
{
Name: flagSampleTypesName,
Help: fmt.Sprintf("choose sample type(s) from: %s", strings.Join(SampleTypeOptions, ", ")),
},
{
Name: flagDurationName,
Help: "number of seconds to run the collection. If 0, the collection will run indefinitely. Ctrl+c to stop.",
Expand All @@ -122,6 +138,10 @@ func getFlagGroups() []app.FlagGroup {
Name: flagPerfEventName,
Help: "perf event to use for native sampling (e.g., cpu-cycles, instructions, cache-misses, branches, context-switches, mem-loads, mem-stores, etc.)",
},
{
Name: flagAsprofArgumentsName,
Help: "arguments to pass to async-profiler, e.g., $ asprof start <these arguments> -i <interval> <pid>.",
},
{
Name: flagMaxDepthName,
Help: "maximum render depth of call stack in flamegraph (0 = no limit)",
Expand Down Expand Up @@ -181,6 +201,15 @@ func validateFlags(cmd *cobra.Command, args []string) error {
if flagMaxDepth < 0 {
return workflow.FlagValidationError(cmd, "max depth must be 0 or greater")
}
// validate sample types
for _, sampleType := range flagSampleTypes {
if !slices.Contains(SampleTypeOptions, sampleType) {
return workflow.FlagValidationError(cmd, fmt.Sprintf("sample type options are: %s", strings.Join(SampleTypeOptions, ", ")))
}
}
if len(flagSampleTypes) == 0 {
return workflow.FlagValidationError(cmd, "at least one sample type must be specified")
}
// common target flags
if err := workflow.ValidateTargetFlags(cmd); err != nil {
return workflow.FlagValidationError(cmd, err.Error())
Expand All @@ -198,11 +227,13 @@ func runCmd(cmd *cobra.Command, args []string) error {
Cmd: cmd,
ReportNamePost: "flame",
ScriptParams: map[string]string{
"Frequency": strconv.Itoa(flagFrequency),
"Duration": strconv.Itoa(flagDuration),
"PIDs": strings.Join(util.IntSliceToStringSlice(flagPids), ","),
"MaxDepth": strconv.Itoa(flagMaxDepth),
"PerfEvent": flagPerfEvent,
"Frequency": strconv.Itoa(flagFrequency),
"Duration": strconv.Itoa(flagDuration),
"PIDs": strings.Join(util.IntSliceToStringSlice(flagPids), ","),
"MaxDepth": strconv.Itoa(flagMaxDepth),
"PerfEvent": flagPerfEvent,
"SampleTypes": strings.Join(flagSampleTypes, ","),
"AsprofArguments": flagAsprofArguments,
},
Tables: tables,
Input: flagInput,
Expand Down
23 changes: 21 additions & 2 deletions cmd/flamegraph/flamegraph_renderers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,8 +229,27 @@ func callStackFrequencyTableHTMLRenderer(tableValues table.TableValues, targetNa
slog.Error("didn't find expected field (Perf Event) in table", slog.String("error", err.Error()))
return out
}
if len(tableValues.Fields[perfEventFieldIndex].Values) == 0 {
slog.Error("no values for perf event field in table")
return out
}
perfEvent := tableValues.Fields[perfEventFieldIndex].Values[0]
out += renderFlameGraph(fmt.Sprintf("Native (%s)", perfEvent), tableValues, "Native Stacks")
out += renderFlameGraph("Java (async-profiler)", tableValues, "Java Stacks")
out += renderFlameGraph(fmt.Sprintf("Native (perf record -e %s)", perfEvent), tableValues, "Native Stacks")

// get the asprof arguments from the table values
asprofArgumentsFieldIndex, err := table.GetFieldIndex("Asprof Arguments", tableValues)
if err != nil {
slog.Error("didn't find expected field (Asprof Arguments) in table", slog.String("error", err.Error()))
return out
}
if len(tableValues.Fields[asprofArgumentsFieldIndex].Values) == 0 {
slog.Error("no values for asprof arguments field in table")
return out
}
asprofArguments := tableValues.Fields[asprofArgumentsFieldIndex].Values[0]
if asprofArguments != "" {
asprofArguments = " " + asprofArguments
}
out += renderFlameGraph(fmt.Sprintf("Java (asprof start%s)", asprofArguments), tableValues, "Java Stacks")
return out
}
20 changes: 19 additions & 1 deletion cmd/flamegraph/flamegraph_tables.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ func flameGraphTableValues(outputs map[string]script.ScriptOutput) []table.Field
{Name: "Java Stacks", Values: []string{javaFoldedFromOutput(outputs)}},
{Name: "Maximum Render Depth", Values: []string{maxRenderDepthFromOutput(outputs)}},
{Name: "Perf Event", Values: []string{perfEventFromOutput(outputs)}},
{Name: "Asprof Arguments", Values: []string{asprofArgumentsFromOutput(outputs)}},
}
return fields
}
Expand Down Expand Up @@ -104,7 +105,6 @@ func nativeFoldedFromOutput(outputs map[string]script.ScriptOutput) string {
}
}
if dwarfFolded == "" && fpFolded == "" {
slog.Warn("no native folded stacks found")
// "event syntax error: 'foo'" indicates that the perf event specified is invalid/unsupported
if strings.Contains(outputs[script.FlameGraphScriptName].Stderr, "event syntax error") {
slog.Error("unsupported perf event specified", slog.String("error", outputs[script.FlameGraphScriptName].Stderr))
Expand Down Expand Up @@ -154,6 +154,24 @@ func perfEventFromOutput(outputs map[string]script.ScriptOutput) string {
return ""
}

func asprofArgumentsFromOutput(outputs map[string]script.ScriptOutput) string {
if outputs[script.FlameGraphScriptName].Stdout == "" {
slog.Warn("collapsed call stack output is empty")
return ""
}
sections := extract.GetSectionsFromOutput(outputs[script.FlameGraphScriptName].Stdout)
if len(sections) == 0 {
slog.Warn("no sections in collapsed call stack output")
return ""
}
for header, content := range sections {
if header == "asprof_arguments" {
return strings.TrimSpace(content)
}
}
return ""
}

// ProcessStacks ...
// [processName][callStack]=count
type ProcessStacks map[string]Stacks
Expand Down
149 changes: 95 additions & 54 deletions internal/script/scripts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1633,43 +1633,67 @@ duration={{.Duration}}
frequency={{.Frequency}}
maxdepth={{.MaxDepth}}
perf_event={{.PerfEvent}}
sample_types={{.SampleTypes}}
read -r -a asprof_arguments <<< "{{.AsprofArguments}}"

# determine which sample types to collect based on the input parameter
sample_native=0
sample_java=0
IFS=',' read -r -a sample_type_array <<< "$sample_types"
for st in "${sample_type_array[@]}"; do
if [[ "$st" == "native" ]]; then
sample_native=1
elif [[ "$st" == "java" ]]; then
sample_java=1
fi
done

ap_interval=0
if [ "$frequency" -ne 0 ]; then
ap_interval=$((1000000000 / frequency))
fi

# Restore original settings
restore_settings() {
echo "$perf_event_paranoid" > /proc/sys/kernel/perf_event_paranoid
echo "$kptr_restrict" > /proc/sys/kernel/kptr_restrict
}

# Function to stop profiling
stop_profiling() {
if [ -n "$perf_fp_pid" ]; then
kill -0 "$perf_fp_pid" 2>/dev/null && kill -INT "$perf_fp_pid"
wait "$perf_fp_pid" || true
if [ $sample_native -eq 1 ]; then
if [ -n "$perf_fp_pid" ]; then
kill -0 "$perf_fp_pid" 2>/dev/null && kill -INT "$perf_fp_pid"
wait "$perf_fp_pid" || true
fi
if [ -n "$perf_dwarf_pid" ]; then
kill -0 "$perf_dwarf_pid" 2>/dev/null && kill -INT "$perf_dwarf_pid"
wait "$perf_dwarf_pid" || true
fi
fi
if [ -n "$perf_dwarf_pid" ]; then
kill -0 "$perf_dwarf_pid" 2>/dev/null && kill -INT "$perf_dwarf_pid"
wait "$perf_dwarf_pid" || true
if [ $sample_java -eq 1 ]; then
for pid in "${java_pids[@]}"; do
async-profiler/bin/asprof stop -o collapsed -f ap_folded_"$pid" "$pid"
done
fi
for pid in "${java_pids[@]}"; do
async-profiler/bin/asprof stop -o collapsed -f ap_folded_"$pid" "$pid"
done
# Restore original settings
echo "$perf_event_paranoid" > /proc/sys/kernel/perf_event_paranoid
echo "$kptr_restrict" > /proc/sys/kernel/kptr_restrict
restore_settings
}

# Function to collapse perf data
collapse_perf_data() {
if [ -f perf_dwarf_data ]; then
perf script -i perf_dwarf_data > perf_dwarf_stacks
stackcollapse-perf perf_dwarf_stacks > perf_dwarf_folded
else
echo "Error: perf_dwarf_data file not found" >&2
fi
if [ -f perf_fp_data ]; then
perf script -i perf_fp_data > perf_fp_stacks
stackcollapse-perf perf_fp_stacks > perf_fp_folded
else
echo "Error: perf_fp_data file not found" >&2
if [ $sample_native -eq 1 ]; then
if [ -f perf_dwarf_data ]; then
perf script -i perf_dwarf_data > perf_dwarf_stacks
stackcollapse-perf perf_dwarf_stacks > perf_dwarf_folded
else
echo "Error: perf_dwarf_data file not found" >&2
fi
if [ -f perf_fp_data ]; then
perf script -i perf_fp_data > perf_fp_stacks
stackcollapse-perf perf_fp_stacks > perf_fp_folded
else
echo "Error: perf_fp_data file not found" >&2
fi
fi
}

Expand All @@ -1678,8 +1702,11 @@ print_results() {
echo "########## maximum depth ##########"
echo "$maxdepth"

echo "########## perf_event ##########"
echo "$perf_event"
echo "########## perf_event ##########"
echo "$perf_event"

echo "########## asprof_arguments ##########"
printf '%s\n' "${asprof_arguments[@]}"

if [ -f perf_dwarf_folded ]; then
echo "########## perf_dwarf ##########"
Expand Down Expand Up @@ -1733,45 +1760,59 @@ if [ -n "$pids" ]; then
fi
else
echo "Error: Process $p is not running." >&2
stop_profiling
restore_settings
exit 1
fi
done
else
mapfile -t java_pids < <(pgrep java)
fi

if [ $sample_native -eq 1 ]; then
# Start profiling with perf in frame pointer mode
if [ -n "$pids" ]; then
perf record -e "$perf_event" -F "$frequency" -p "$pids" -g -o perf_fp_data -m 129 &
else
perf record -e "$perf_event" -F "$frequency" -a -g -o perf_fp_data -m 129 &
fi
perf_fp_pid=$!
if ! kill -0 $perf_fp_pid 2>/dev/null; then
echo "Failed to start perf record in frame pointer mode" >&2
stop_profiling
exit 1
fi
if [ -n "$pids" ]; then
perf record -e "$perf_event" -F "$frequency" -p "$pids" -g -o perf_fp_data -m 129 &
else
perf record -e "$perf_event" -F "$frequency" -a -g -o perf_fp_data -m 129 &
fi
perf_fp_pid=$!
if ! kill -0 $perf_fp_pid 2>/dev/null; then
echo "Failed to start perf record in frame pointer mode" >&2
stop_profiling
exit 1
fi

# Start profiling with perf in dwarf mode
if [ -n "$pids" ]; then
perf record -e "$perf_event" -F "$frequency" -p "$pids" -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 &
else
perf record -e "$perf_event" -F "$frequency" -a -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 &
fi
perf_dwarf_pid=$!
if ! kill -0 $perf_dwarf_pid 2>/dev/null; then
echo "Failed to start perf record in dwarf mode" >&2
stop_profiling
exit 1
fi
# Start profiling with perf in dwarf mode
if [ -n "$pids" ]; then
perf record -e "$perf_event" -F "$frequency" -p "$pids" -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 &
else
perf record -e "$perf_event" -F "$frequency" -a -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 &
fi
perf_dwarf_pid=$!
if ! kill -0 $perf_dwarf_pid 2>/dev/null; then
echo "Failed to start perf record in dwarf mode" >&2
stop_profiling
exit 1
fi
fi # if sample_native

# Start profiling Java with async-profiler for each Java PID
for pid in "${java_pids[@]}"; do
java_cmds+=("$(tr '\000' ' ' < /proc/"$pid"/cmdline)")
async-profiler/bin/asprof start -i "$ap_interval" -F probesp+vtable "$pid"
done
if [ $sample_java -eq 1 ]; then
if [ ${#java_pids[@]} -eq 0 ]; then
echo "Warning: Java sampling was requested, but no Java processes were found; skipping Java profiling." >&2
else
# Start profiling Java with async-profiler for each Java PID
for pid in "${java_pids[@]}"; do
java_cmds+=("$(tr '\000' ' ' < /proc/"$pid"/cmdline)")
async-profiler/bin/asprof start "${asprof_arguments[@]}" -i "$ap_interval" "$pid"
asprof_pid=$!
if ! kill -0 $asprof_pid 2>/dev/null; then
echo "Failed to start async-profiler for PID $pid" >&2
stop_profiling
exit 1
fi
done
fi
fi

# profiling has been started, set up trap to finalize on interrupt
trap finalize INT TERM EXIT
Expand Down