diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 3c05347bf17a5..70f19034ac5eb 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -726,6 +726,14 @@ CINDEX_LINKAGE void clang_experimental_DependencyScannerReproducerOptions_dispose( CXDependencyScannerReproducerOptions); +/** + * Specify the object store and action cache databases, and CAS options for + * generating a reproducer. Should be used if the original compilation uses CAS. + */ +CINDEX_LINKAGE void +clang_experimental_DependencyScannerReproducerOptions_setCASOptions( + CXDependencyScannerReproducerOptions, CXCASDatabases, CXCASOptions); + /** * Generates a reproducer to compile a requested file with required modules. * diff --git a/clang/test/Modules/reproducer-with-module-dependencies.c b/clang/test/Modules/reproducer-with-module-dependencies.c index 906e55e5acd9b..84f394a63e3e9 100644 --- a/clang/test/Modules/reproducer-with-module-dependencies.c +++ b/clang/test/Modules/reproducer-with-module-dependencies.c @@ -1,5 +1,6 @@ // Test generating a reproducer for a modular build where required modules are // built explicitly as separate steps. +// REQUIRES: shell // RUN: rm -rf %t // RUN: split-file %s %t @@ -7,23 +8,40 @@ // // RUN: c-index-test core -gen-deps-reproducer -working-dir %t \ // RUN: -- clang-executable -c %t/reproducer.c -o %t/reproducer.o \ -// RUN: -fmodules -fmodules-cache-path=%t \ +// RUN: -fmodules -fmodules-cache-path=%t.modulecache \ // RUN: -ivfsoverlay %t/existing.yaml -I /virtual | FileCheck %t/reproducer.c // Test a failed attempt at generating a reproducer. // RUN: not c-index-test core -gen-deps-reproducer -working-dir %t \ // RUN: -- clang-executable -c %t/failed-reproducer.c -o %t/reproducer.o \ -// RUN: -fmodules -fmodules-cache-path=%t 2>&1 | FileCheck %t/failed-reproducer.c +// RUN: -fmodules -fmodules-cache-path=%t.modulecache 2>&1 | FileCheck %t/failed-reproducer.c // Test the content of a reproducer script. // RUN: c-index-test core -gen-deps-reproducer -working-dir %t -o %t/repro-content \ // RUN: -- clang-executable -c %t/reproducer.c -o %t/reproducer.o \ -// RUN: -fmodules -fmodules-cache-path=%t \ +// RUN: -fmodules -fmodules-cache-path=%t.modulecache \ // RUN: -DMACRO="\$foo" \ // RUN: -ivfsoverlay %t/existing.yaml -I /virtual \ // RUN: -MMD -MT dependencies -MF %t/deps.d // RUN: FileCheck %t/script-expectations.txt --input-file %t/repro-content/reproducer.sh +// Test the content of a reproducer script with CAS enabled. +// RUN: c-index-test core -gen-deps-reproducer -working-dir %t -cas-path %t/cas -o %t/repro-cas-content \ +// RUN: -- %clang -c %t/reproducer.c -o %t/reproducer.o \ +// RUN: -fmodules -fmodules-cache-path=%t.modulecache.cas \ +// RUN: -I %t/include +// RUN: FileCheck %t/cas-script-expectations.txt --input-file %t/repro-cas-content/reproducer.sh + +// Verify can reproduce the original compilations with the files captured by reproducers. +// RUN: rm -rf %t.modulecache +// RUN: rm -rf %t.modulecache.cas +// RUN: rm -rf %t/cas +// RUN: rm %t/include/modular-header.h +// RUN: cd %t/repro-content +// RUN: env CLANG=%clang bash ./reproducer.sh +// RUN: cd %t/repro-cas-content +// RUN: bash ./reproducer.sh + //--- include/modular-header.h void fn_in_modular_header(void); @@ -76,3 +94,8 @@ CHECK: -ivfsoverlay "reproducer.cache/vfs/vfs.yaml" CHECK: "-ivfsoverlay" "{{.*}}/existing.yaml" CHECK: MACRO=\$foo CHECK: "-dependency-file" "reproducer.cache/deps.d" + +//--- cas-script-expectations.txt +CHECK: -fcas-path "reproducer.cache/cas" +CHECK: "-fcas-include-tree" "llvmcas:// +CHECK: "-fmodule-file-cache-key" "Test-{{.*}}.pcm" "llvmcas:// diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 19d3fad09dd29..9dd1e3f98a523 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -955,7 +955,8 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, static int generateDepsReproducer(ArrayRef Args, std::string WorkingDirectory, - std::string ReproLocation) { + std::string ReproLocation, + CXCASDatabases DBs) { CXDependencyScannerReproducerOptions Opts = clang_experimental_DependencyScannerReproducerOptions_create( Args.size(), Args.data(), /*ModuleName=*/nullptr, @@ -965,6 +966,8 @@ static int generateDepsReproducer(ArrayRef Args, auto DisposeOpts = llvm::make_scope_exit([&] { clang_experimental_DependencyScannerReproducerOptions_dispose(Opts); }); + clang_experimental_DependencyScannerReproducerOptions_setCASOptions(Opts, DBs, + nullptr); CXString MessageString; auto DisposeMessageString = llvm::make_scope_exit([&]() { clang_disposeString(MessageString); @@ -1611,7 +1614,7 @@ int indextest_core_main(int argc, const char **argv) { return 1; } return generateDepsReproducer(CompArgs, options::WorkingDir, - options::OutputFile); + options::OutputFile, DBs); } if (options::Action == ActionType::UploadCachedJob) { diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 4c5168a550477..c280b9b740a8d 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -694,6 +694,9 @@ struct DependencyScannerReproducerOptions { std::optional WorkingDirectory; std::optional ReproducerLocation; bool UseUniqueReproducerName; + CASOptions CASOpts; + std::shared_ptr CAS; + std::shared_ptr Cache; DependencyScannerReproducerOptions(int argc, const char *const *argv, const char *ModuleName, @@ -750,6 +753,20 @@ clang_experimental_DependencyScannerReproducerOptions_create( UseUniqueReproducerName}); } +void clang_experimental_DependencyScannerReproducerOptions_setCASOptions( + CXDependencyScannerReproducerOptions CXOptions, CXCASDatabases CDBs, + CXCASOptions CASOpts) { + DependencyScannerReproducerOptions &Opts = *unwrap(CXOptions); + if (CDBs) { + cas::WrappedCASDatabases &DBs = *cas::unwrap(CDBs); + Opts.CASOpts = DBs.CASOpts; + Opts.CAS = DBs.CAS; + Opts.Cache = DBs.Cache; + } + if (CASOpts) + Opts.CASOpts = *cas::unwrap(CASOpts); +} + void clang_experimental_DependencyScannerReproducerOptions_dispose( CXDependencyScannerReproducerOptions Options) { delete unwrap(Options); @@ -773,11 +790,18 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( return Report(CXError_InvalidArguments) << "non-unique reproducer is allowed only in a custom location"; - CASOptions CASOpts; + std::shared_ptr UpstreamCAS = Opts.CAS; + bool IsReproducerCASBased(UpstreamCAS); DependencyScanningService DepsService( - ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Full, - CASOpts, /*CAS=*/nullptr, /*ActionCache=*/nullptr); - DependencyScanningTool DepsTool(DepsService); + ScanningMode::DependencyDirectivesScan, + IsReproducerCASBased ? ScanningOutputFormat::FullIncludeTree + : ScanningOutputFormat::Full, + Opts.CASOpts, UpstreamCAS, Opts.Cache); + IntrusiveRefCntPtr FS = + llvm::vfs::createPhysicalFileSystem(); + if (UpstreamCAS) + FS = llvm::cas::createCASProvidingFileSystem(UpstreamCAS, std::move(FS)); + DependencyScanningTool DepsTool(DepsService, FS); llvm::SmallString<128> ReproScriptPath; int ScriptFD; @@ -844,7 +868,7 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( // it is easier to run the reproducer with a different compiler and to // simplify running an individual command manually. std::string ReproExecutable = "\"${CLANG:-" + Opts.BuildArgs.front() + "}\""; - auto PrintArguments = [&ReproExecutable, &FileCacheName, + auto PrintArguments = [IsReproducerCASBased, &ReproExecutable, &FileCacheName, &ClangOpts](llvm::raw_fd_ostream &OS, ArrayRef Arguments, bool RedirectOutput) { @@ -856,7 +880,8 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( ClangOpts.ParseArgs(CharArgs, MissingArgIndex, MissingArgCount, llvm::opt::Visibility(options::CC1Option)); - bool DidAddVFSOverlay = false; + // CAS-based reproducer doesn't use VFS overlays. + bool DidAddVFSOverlay = IsReproducerCASBased; OS << ReproExecutable; for (const llvm::opt::Arg *Arg : ParsedArgs) { const llvm::opt::Option &Opt = Arg->getOption(); @@ -866,6 +891,10 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( DidAddVFSOverlay = true; } } + if (Opt.matches(options::OPT_fcas_path)) { + OS << " -fcas-path \"" << FileCacheName << "/cas\""; + continue; + } bool IsOutputArg = Opt.matches(options::OPT_o) || Opt.matches(options::OPT_dependency_file); llvm::opt::ArgStringList OutArgs; @@ -899,6 +928,60 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( auto RealFS = llvm::vfs::getRealFileSystem(); RealFS->setCurrentWorkingDirectory(*Opts.WorkingDirectory); + if (IsReproducerCASBased) { + SmallString<128> CASPath = FileCachePath; + llvm::sys::path::append(CASPath, "cas"); + clang::CASOptions ReproducerCASOpts; + ReproducerCASOpts.CASPath = CASPath.str(); + ReproducerCASOpts.PluginPath = Opts.CASOpts.PluginPath; + ReproducerCASOpts.PluginOptions = Opts.CASOpts.PluginOptions; + auto DBsOrErr = ReproducerCASOpts.getOrCreateDatabases(); + if (!DBsOrErr) + return ReportFailure() << "failed to create a CAS database\n" + << toString(DBsOrErr.takeError()); + std::shared_ptr ReproCAS = DBsOrErr->first; + + auto transplantCASIncludeTree = + [UpstreamCAS, ReproCAS]( + const std::optional &IncludeTreeID) -> llvm::Error { + if (!IncludeTreeID.has_value()) + // Missing `IncludeTreeID` likely indicates a problem but ignore it, so + // can capture enough data to reproduce it later. + return llvm::Error::success(); + auto IDOrErr = UpstreamCAS->parseID(*IncludeTreeID); + if (!IDOrErr) + return llvm::make_error( + "failure to parse include tree id '" + *IncludeTreeID + + "':" + toString(IDOrErr.takeError()), + llvm::inconvertibleErrorCode()); + std::optional UpstreamRef = + UpstreamCAS->getReference(*IDOrErr); + if (!UpstreamRef.has_value()) + return llvm::make_error( + "missing include tree with ID '" + *IncludeTreeID + + "' in the provided CAS object storage", + llvm::inconvertibleErrorCode()); + auto ReproRefOrErr = ReproCAS->importObject(*UpstreamCAS, *UpstreamRef); + if (!ReproRefOrErr) + return llvm::make_error( + "failure to import an include tree with id '" + *IncludeTreeID + + "':" + toString(ReproRefOrErr.takeError()), + llvm::inconvertibleErrorCode()); + return llvm::Error::success(); + }; + + if (auto Err = transplantCASIncludeTree(TU.IncludeTreeID)) + return ReportFailure() + << "failed to transplant a translation unit include tree due to " + << toString(std::move(Err)); + for (const ModuleDeps &ModuleDep : TU.ModuleGraph) { + if (auto Err = transplantCASIncludeTree(ModuleDep.IncludeTreeID)) + return ReportFailure() + << "failed to transplant a module '" + ModuleDep.ID.ModuleName + + "' include tree due to " + << toString(std::move(Err)); + } +} else { SmallString<128> VFSCachePath = FileCachePath; llvm::sys::path::append(VFSCachePath, "vfs"); std::string VFSCachePathStr = VFSCachePath.str().str(); @@ -919,6 +1002,7 @@ enum CXErrorCode clang_experimental_DependencyScanner_generateReproducer( llvm::sys::path::append(VFSOverlayPath, "vfs.yaml"); if (FileCollector.writeMapping(VFSOverlayPath)) return ReportFailure() << "failed to write a VFS overlay mapping"; +} return Report(CXError_Success) << "Created a reproducer. Sources and associated run script(s) are " diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 703d7107f6402..d3121dd11d2d9 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -612,6 +612,7 @@ LLVM_21 { LLVM_22 { global: clang_experimental_DependencyScannerServiceOptions_setCacheNegativeStats; + clang_experimental_DependencyScannerReproducerOptions_setCASOptions; }; # Example of how to add a new symbol version entry. If you do add a new symbol