diff --git a/coroutine_test.go b/coroutine_test.go new file mode 100644 index 0000000..1cff30a --- /dev/null +++ b/coroutine_test.go @@ -0,0 +1,289 @@ +//===- coroutine_test.go - Tests for coroutine lowering ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +package llvm + +import ( + "strconv" + "strings" + "testing" +) + +func TestSwitchedResumeCoroutinePassPipelines(t *testing.T) { + majorVersion, err := strconv.Atoi(strings.SplitN(Version, ".", 2)[0]) + if err != nil { + t.Fatalf("could not parse LLVM version %q: %v", Version, err) + } + + InitializeAllTargetInfos() + InitializeAllTargets() + InitializeAllTargetMCs() + InitializeAllAsmPrinters() + triple := DefaultTargetTriple() + target, err := GetTargetFromTriple(triple) + if err != nil { + t.Fatal(err) + } + tm := target.CreateTargetMachine( + triple, "", "", CodeGenLevelDefault, RelocDefault, CodeModelDefault, + ) + defer tm.Dispose() + + explicitPipeline := "coro-early,cgscc(coro-split),coro-cleanup" + if majorVersion == 14 { + // In LLVM 14, coro-early and coro-cleanup are function passes. Newer + // releases expose them as module passes and accept the unnested form. + explicitPipeline = "function(coro-early),cgscc(coro-split),function(coro-cleanup)" + } + for _, test := range []struct { + name string + pipeline string + }{ + {name: "explicit coroutine pipeline", pipeline: explicitPipeline}, + {name: "production O0 pipeline", pipeline: "default"}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := NewContext() + defer ctx.Dispose() + mod := buildSwitchedResumeCoroutine(t, ctx, tm, majorVersion) + defer mod.Dispose() + assertSwitchedResumeCoroutineFrontendState(t, mod, majorVersion) + + if err := VerifyModule(mod, ReturnStatusAction); err != nil { + t.Fatalf("coroutine module did not verify before %s: %v\n%s", test.pipeline, err, mod.String()) + } + + options := NewPassBuilderOptions() + defer options.Dispose() + options.SetVerifyEach(true) + if err := mod.RunPasses(test.pipeline, tm, options); err != nil { + t.Fatalf("could not run %s: %v\n%s", test.pipeline, err, mod.String()) + } + + if err := VerifyModule(mod, ReturnStatusAction); err != nil { + t.Fatalf("coroutine module did not verify after %s: %v\n%s", test.pipeline, err, mod.String()) + } + assertSwitchedResumeCoroutineLowered(t, mod, test.pipeline) + }) + } +} + +func buildSwitchedResumeCoroutine(t *testing.T, ctx Context, tm TargetMachine, majorVersion int) Module { + t.Helper() + + mod := ctx.NewModule("switched-resume-coroutine") + returned := false + defer func() { + if !returned { + mod.Dispose() + } + }() + mod.SetTarget(tm.Triple()) + targetData := tm.CreateTargetData() + mod.SetDataLayout(targetData.String()) + pointerSize := targetData.PointerSize() + targetData.Dispose() + + i1 := ctx.Int1Type() + i8 := ctx.Int8Type() + i32 := ctx.Int32Type() + ptr := PointerType(i8, 0) + void := ctx.VoidType() + falseValue := ConstInt(i1, 0, false) + nullPointer := ConstPointerNull(ptr) + var sizeType Type + switch pointerSize { + case 4: + sizeType = i32 + case 8: + sizeType = ctx.Int64Type() + default: + t.Fatalf("unsupported target pointer size %d", pointerSize) + } + + fn := AddFunction(mod, "stackless", FunctionType(ptr, nil, false)) + if majorVersion == 14 { + // LLVM 14 models this as a frontend-to-CoroEarly state machine: "0" + // means unprepared input, while "1" is reserved for prepared IR. + fn.AddFunctionAttr(ctx.CreateStringAttribute("coroutine.presplit", "0")) + } else { + kind := AttributeKindID("presplitcoroutine") + if kind == 0 { + t.Fatal("could not look up presplitcoroutine attribute") + } + fn.AddFunctionAttr(ctx.CreateEnumAttribute(kind, 0)) + } + + frameAllocType := FunctionType(ptr, []Type{sizeType, sizeType}, false) + frameAlloc := AddFunction(mod, "coro_frame_alloc", frameAllocType) + freeType := FunctionType(void, []Type{ptr}, false) + free := AddFunction(mod, "coro_frame_free", freeType) + + entry := AddBasicBlock(fn, "entry") + allocate := AddBasicBlock(fn, "allocate") + begin := AddBasicBlock(fn, "begin") + initialSuspend := AddBasicBlock(fn, "initial.suspend") + body := AddBasicBlock(fn, "body") + finalSuspend := AddBasicBlock(fn, "final.suspend") + finalResumeUnreachable := AddBasicBlock(fn, "final.resume.unreachable") + cleanup := AddBasicBlock(fn, "cleanup") + deallocate := AddBasicBlock(fn, "deallocate") + end := AddBasicBlock(fn, "end") + + ids := make(map[string]int) + for _, name := range []string{ + "llvm.coro.id", + "llvm.coro.alloc", + "llvm.coro.size", + "llvm.coro.align", + "llvm.coro.begin", + "llvm.coro.save", + "llvm.coro.suspend", + "llvm.coro.free", + "llvm.coro.end", + } { + ids[name] = LookupIntrinsicID(name) + if ids[name] == 0 { + t.Fatalf("could not look up %s intrinsic", name) + } + } + + builder := ctx.NewBuilder() + defer builder.Dispose() + + createIntrinsic := func(ret Type, name string, args []Value, resultName string) Value { + value := builder.CreateIntrinsic(ret, ids[name], args, resultName) + if value.IsNil() { + t.Fatalf("could not construct %s intrinsic", name) + } + return value + } + createSuspend := func(block BasicBlock, handle Value, final bool, resume, destroy BasicBlock) { + builder.SetInsertPointAtEnd(block) + saved := createIntrinsic(ctx.TokenType(), "llvm.coro.save", []Value{handle}, "save") + finalValue := ConstInt(i1, 0, false) + if final { + finalValue = ConstInt(i1, 1, false) + } + suspended := createIntrinsic(i8, "llvm.coro.suspend", []Value{saved, finalValue}, "suspend") + dispatch := builder.CreateSwitch(suspended, end, 2) + dispatch.AddCase(ConstInt(i8, 0, false), resume) + dispatch.AddCase(ConstInt(i8, 1, false), destroy) + } + + builder.SetInsertPointAtEnd(entry) + // Zero requests LLVM's default allocation memory alignment guarantee of + // twice the target pointer size. + allocationAlignmentGuarantee := ConstInt(i32, 0, false) + id := createIntrinsic(ctx.TokenType(), "llvm.coro.id", []Value{ + allocationAlignmentGuarantee, + nullPointer, + nullPointer, + nullPointer, + }, "id") + needsAllocation := createIntrinsic(i1, "llvm.coro.alloc", []Value{id}, "needs.alloc") + builder.CreateCondBr(needsAllocation, allocate, begin) + + builder.SetInsertPointAtEnd(allocate) + frameSize := createIntrinsic(sizeType, "llvm.coro.size", nil, "frame.size") + frameAlignment := createIntrinsic(sizeType, "llvm.coro.align", nil, "frame.align") + minimumAlignment := ConstInt(sizeType, uint64(2*pointerSize), false) + belowMinimum := builder.CreateICmp(IntULT, frameAlignment, minimumAlignment, "below.minimum.alignment") + allocationAlignment := builder.CreateSelect( + belowMinimum, minimumAlignment, frameAlignment, "allocation.alignment", + ) + allocatedMemory := builder.CreateCall( + frameAllocType, frameAlloc, []Value{frameSize, allocationAlignment}, "allocated.memory", + ) + builder.CreateBr(begin) + + builder.SetInsertPointAtEnd(begin) + frameMemory := builder.CreatePHI(ptr, "frame.memory") + frameMemory.AddIncoming( + []Value{nullPointer, allocatedMemory}, + []BasicBlock{entry, allocate}, + ) + handle := createIntrinsic(ptr, "llvm.coro.begin", []Value{id, frameMemory}, "handle") + builder.CreateBr(initialSuspend) + + createSuspend(initialSuspend, handle, false, body, cleanup) + createSuspend(body, handle, false, finalSuspend, cleanup) + createSuspend(finalSuspend, handle, true, finalResumeUnreachable, cleanup) + + builder.SetInsertPointAtEnd(finalResumeUnreachable) + builder.CreateUnreachable() + + builder.SetInsertPointAtEnd(cleanup) + frameToFree := createIntrinsic(ptr, "llvm.coro.free", []Value{id, handle}, "frame.to.free") + hasFrameToFree := builder.CreateIsNotNull(frameToFree, "has.frame.to.free") + builder.CreateCondBr(hasFrameToFree, deallocate, end) + + builder.SetInsertPointAtEnd(deallocate) + builder.CreateCall(freeType, free, []Value{frameToFree}, "") + builder.CreateBr(end) + + builder.SetInsertPointAtEnd(end) + endArgs := []Value{handle, falseValue} + endType := i1 + endName := "ended" + if majorVersion >= 18 { + endArgs = append(endArgs, ctx.ConstTokenNone()) + } + if majorVersion >= 22 { + endType = void + endName = "" + } + createIntrinsic(endType, "llvm.coro.end", endArgs, endName) + builder.CreateRet(handle) + + returned = true + return mod +} + +func assertSwitchedResumeCoroutineFrontendState(t *testing.T, mod Module, majorVersion int) { + t.Helper() + + text := mod.String() + if majorVersion == 14 { + if !strings.Contains(text, `"coroutine.presplit"="0"`) { + t.Fatalf("LLVM 14 frontend coroutine is not marked unprepared for CoroEarly:\n%s", text) + } + } else if !strings.Contains(text, "presplitcoroutine") { + t.Fatalf("frontend coroutine lacks presplitcoroutine attribute:\n%s", text) + } + if !strings.Contains(text, "%allocation.alignment = select i1") { + t.Fatalf("frame allocator does not normalize coro.align to the coro.id guarantee:\n%s", text) + } + for _, line := range strings.Split(text, "\n") { + if strings.Contains(line, "call") && strings.Contains(line, "@coro_frame_alloc") { + if !strings.Contains(line, "%allocation.alignment") { + t.Fatalf("frame allocator does not receive the normalized alignment: %s\n%s", line, text) + } + return + } + } + t.Fatalf("frontend coroutine does not call the frame allocator:\n%s", text) +} + +func assertSwitchedResumeCoroutineLowered(t *testing.T, mod Module, pipeline string) { + t.Helper() + + for _, name := range []string{"stackless", "stackless.resume", "stackless.destroy"} { + fn := mod.NamedFunction(name) + if fn.IsNil() || fn.BasicBlocksCount() == 0 { + t.Fatalf("%s did not produce coroutine function %q:\n%s", pipeline, name, mod.String()) + } + } + + text := mod.String() + for _, line := range strings.Split(text, "\n") { + if strings.Contains(line, " call ") && strings.Contains(line, "@llvm.coro.") { + t.Fatalf("%s left coroutine intrinsic call after lowering: %s\n%s", pipeline, line, text) + } + } +}