-
Notifications
You must be signed in to change notification settings - Fork 38
Add export_to_reactant_script and automatic debug zip on pass pipeline failure #1942
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
f0d643d
2705612
1b64083
79ad3f9
ee78c93
8227305
8c3bec2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -4,6 +4,7 @@ using Reactant_jll | |||||||||||
| using Libdl: dlsym | ||||||||||||
| using LinearAlgebra: BlasInt | ||||||||||||
| using Functors: Functors | ||||||||||||
| import p7zip_jll: p7zip | ||||||||||||
|
|
||||||||||||
| import ..Reactant: | ||||||||||||
| Reactant, | ||||||||||||
|
|
@@ -1355,6 +1356,102 @@ end | |||||||||||
|
|
||||||||||||
| const context_gc_vector = Dict{MLIR.IR.Context,Vector{Union{TracedRArray,TracedRNumber}}}() | ||||||||||||
|
|
||||||||||||
| """ | ||||||||||||
| create_pass_failure_zip(mod, f, args, pass_pipeline_key, error_msg) | ||||||||||||
|
|
||||||||||||
| Create a zip file containing the unoptimized IR and a Julia script for reproducing the issue. | ||||||||||||
| This is automatically called when a pass pipeline fails during compilation. | ||||||||||||
|
|
||||||||||||
| Returns the path to the created zip file. | ||||||||||||
| """ | ||||||||||||
| function create_pass_failure_zip( | ||||||||||||
| mod::MLIR.IR.Module, f, args, pass_pipeline_key::String, error_msg::String | ||||||||||||
| ) | ||||||||||||
| try | ||||||||||||
| # Create a temporary directory for the files | ||||||||||||
| temp_dir = mktempdir(; prefix="reactant_failure_", cleanup=false) | ||||||||||||
|
|
||||||||||||
| # Save the unoptimized IR | ||||||||||||
| mlir_path = joinpath(temp_dir, "unoptimized_ir.mlir") | ||||||||||||
| open(mlir_path, "w") do io | ||||||||||||
| println(io, "// Pass pipeline that failed: ", pass_pipeline_key) | ||||||||||||
| println(io, "// Error message: ", error_msg) | ||||||||||||
| println(io) | ||||||||||||
| show(IOContext(io, :debug => true), mod) | ||||||||||||
| end | ||||||||||||
|
|
||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| # Try to export inputs and create a Julia script using Serialization | ||||||||||||
| function_name = string(f) | ||||||||||||
| script_path = nothing | ||||||||||||
| try | ||||||||||||
| # Check if NPZ is available for serialization | ||||||||||||
| if Reactant.Serialization.serialization_supported(Val(:NPZ)) | ||||||||||||
| script_path = Reactant.Serialization.export_to_reactant_script( | ||||||||||||
| f, args...; output_dir=temp_dir, function_name=function_name | ||||||||||||
| ) | ||||||||||||
| end | ||||||||||||
| catch e | ||||||||||||
| @debug "Could not create Julia script for reproduction" exception = e | ||||||||||||
| end | ||||||||||||
|
|
||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| # Create README with instructions | ||||||||||||
| readme_path = joinpath(temp_dir, "README.md") | ||||||||||||
| open(readme_path, "w") do io | ||||||||||||
| println(io, "# Reactant Compilation Failure Report") | ||||||||||||
| println(io) | ||||||||||||
| println(io, "This archive contains information about a failed Reactant compilation.") | ||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| println(io) | ||||||||||||
| println(io, "## Contents") | ||||||||||||
| println(io) | ||||||||||||
| println(io, "- `unoptimized_ir.mlir`: The MLIR module before optimization passes") | ||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| println(io, "- `README.md`: This file") | ||||||||||||
| if script_path !== nothing | ||||||||||||
| println(io, "- `$(function_name).jl`: Julia script for reproduction") | ||||||||||||
| println(io, "- `$(function_name)*.mlir`: Exported HLO code") | ||||||||||||
| println(io, "- `$(function_name)*_inputs.npz`: Input data") | ||||||||||||
| end | ||||||||||||
| println(io) | ||||||||||||
| println(io, "## Error Information") | ||||||||||||
| println(io) | ||||||||||||
| println(io, "**Pass Pipeline Key**: `$(pass_pipeline_key)`") | ||||||||||||
| println(io) | ||||||||||||
| println(io, "**Error Message**:") | ||||||||||||
| println(io, "```") | ||||||||||||
| println(io, error_msg) | ||||||||||||
| println(io, "```") | ||||||||||||
| println(io) | ||||||||||||
| println(io, "## How to Report") | ||||||||||||
| println(io) | ||||||||||||
| println(io, "1. Upload this zip file to a file sharing service") | ||||||||||||
| println(io, "2. Open an issue at https://github.com/EnzymeAD/Reactant.jl/issues") | ||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| println(io, "3. Include the link to the uploaded zip file in your issue") | ||||||||||||
| println(io, "4. Describe what you were trying to do when the error occurred") | ||||||||||||
| println(io) | ||||||||||||
| println(io, "## Debugging") | ||||||||||||
| println(io) | ||||||||||||
| println(io, "You can inspect the `unoptimized_ir.mlir` file to see the IR before it failed.") | ||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| if script_path !== nothing | ||||||||||||
| println(io, "You can also try running the `$(function_name).jl` script to reproduce the issue.") | ||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| end | ||||||||||||
| end | ||||||||||||
|
|
||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| # Create the zip file | ||||||||||||
| zip_path = temp_dir * ".zip" | ||||||||||||
| # Note: temp_files are passed as command arguments (not via shell expansion) | ||||||||||||
| # which prevents shell injection even if paths contain special characters | ||||||||||||
| temp_files = readdir(temp_dir; join=true) | ||||||||||||
| run(pipeline(`$(p7zip()) a -tzip $(zip_path) $(temp_files...)`, devnull)) | ||||||||||||
|
|
||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| # Clean up the temp directory (but keep the zip) | ||||||||||||
| rm(temp_dir; recursive=true, force=true) | ||||||||||||
|
|
||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| return zip_path | ||||||||||||
| catch e | ||||||||||||
| @error "Failed to create debug zip file" exception = e | ||||||||||||
| return nothing | ||||||||||||
| end | ||||||||||||
| end | ||||||||||||
|
|
||||||||||||
| # helper for debug purposes: String -> Text | ||||||||||||
| function run_pass_pipeline_on_source(source, pass_pipeline; enable_verifier=true) | ||||||||||||
| return MLIR.IR.with_context() do ctx | ||||||||||||
|
|
@@ -1425,16 +1522,42 @@ function compile_mlir(f, args; client=nothing, drop_unsupported_attributes=false | |||||||||||
| mod = MLIR.IR.Module(MLIR.IR.Location()) | ||||||||||||
|
|
||||||||||||
| compile_options, kwargs = __get_compile_options_and_kwargs(; kwargs...) | ||||||||||||
| mlir_fn_res = compile_mlir!( | ||||||||||||
| mod, | ||||||||||||
| f, | ||||||||||||
| args, | ||||||||||||
| compile_options; | ||||||||||||
| backend, | ||||||||||||
| runtime=XLA.runtime(client), | ||||||||||||
| client, | ||||||||||||
| kwargs..., | ||||||||||||
| ) | ||||||||||||
|
|
||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| # Wrap compile_mlir! to catch pass pipeline failures | ||||||||||||
| mlir_fn_res = try | ||||||||||||
| compile_mlir!( | ||||||||||||
| mod, | ||||||||||||
| f, | ||||||||||||
| args, | ||||||||||||
| compile_options; | ||||||||||||
| backend, | ||||||||||||
| runtime=XLA.runtime(client), | ||||||||||||
| client, | ||||||||||||
| kwargs..., | ||||||||||||
| ) | ||||||||||||
| catch e | ||||||||||||
| # Check if this is a pass pipeline failure | ||||||||||||
| error_msg = string(e) | ||||||||||||
| if contains(error_msg, "failed to run pass manager") | ||||||||||||
| # Create a debug zip file with the unoptimized IR | ||||||||||||
| zip_path = create_pass_failure_zip(mod, f, args, "compilation", error_msg) | ||||||||||||
| if zip_path !== nothing | ||||||||||||
| error( | ||||||||||||
| "Compilation failed during pass pipeline execution. " * | ||||||||||||
| "A debug zip file has been created at: $(zip_path)\n" * | ||||||||||||
| "Please upload this file when reporting the issue at: " * | ||||||||||||
| "https://github.com/EnzymeAD/Reactant.jl/issues\n" * | ||||||||||||
| "Original error: $(error_msg)" | ||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [JuliaFormatter] reported by reviewdog 🐶
Suggested change
|
||||||||||||
| ) | ||||||||||||
| else | ||||||||||||
| # If we couldn't create the zip, just rethrow the original error | ||||||||||||
| rethrow() | ||||||||||||
| end | ||||||||||||
| else | ||||||||||||
| # Not a pass pipeline failure, rethrow | ||||||||||||
| rethrow() | ||||||||||||
| end | ||||||||||||
| end | ||||||||||||
|
|
||||||||||||
| # Attach a name, and partitioning attributes to the module | ||||||||||||
| __add_mhlo_attributes_and_name!( | ||||||||||||
|
|
||||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[JuliaFormatter] reported by reviewdog 🐶