Skip to content

[swift][caching] Load swift module from CAS #11983

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
cachemeifyoucan wants to merge 1 commit into
base: swift/release/6.3
Choose a base branch
from
Draft

[swift][caching] Load swift module from CAS #11983

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 53 additions & 30 deletions lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1923,8 +1923,8 @@ void SwiftASTContext::AddExtraClangCC1Args(
const std::vector<std::pair<std::string, bool>> framework_search_paths,
std::vector<std::string> &dest) {
clang::CompilerInvocation invocation;
std::vector<std::string> default_paths = {"/usr/include",
"/user/local/include"};
std::vector<std::string> default_paths = {
"/usr/include", "/usr/local/include", "/usr/lib/swift/shims"};
llvm::SmallVector<const char *> clangArgs;
clangArgs.reserve(source.size() + module_search_paths.size() * 2 +
framework_search_paths.size() * 2 +
Expand Down Expand Up @@ -1980,34 +1980,42 @@ void SwiftASTContext::AddExtraClangCC1Args(
invocation.getHeaderSearchOpts().ModuleCachePath =
GetCompilerInvocation().getClangModuleCachePath().str();

bool use_cas_module = m_cas && m_action_cache;
if (use_cas_module) {
// Load from CAS.
invocation.getCASOpts().CASPath = GetCASOptions().CASOpts.CASPath;
invocation.getCASOpts().PluginPath = GetCASOptions().CASOpts.PluginPath;
invocation.getCASOpts().PluginOptions =
GetCASOptions().CASOpts.PluginOptions;

use_cas_module = llvm::all_of(
invocation.getFrontendOpts().ModuleCacheKeys, [&](const auto &entry) {
bool exist = IsModuleAvailableInCAS(entry.second);
if (!exist)
HEALTH_LOG_PRINTF("module '%s' cannot be load "
"from CAS using key: %s, fallback to "
"load from file system",
entry.first.c_str(), entry.second.c_str());
return exist;
});
}

if (!use_cas_module) {
// Clear module cache key and other CAS options to load modules from disk
// directly.
auto ClearCASOptions = [&]() {
invocation.getFrontendOpts().ModuleCacheKeys.clear();
invocation.getCASOpts() = clang::CASOptions();

// Ignore CAS info inside modules when loading.
invocation.getFrontendOpts().ModuleLoadIgnoreCAS = true;
};

bool use_cas_modules = m_cas && m_action_cache &&
!invocation.getFrontendOpts().ModuleCacheKeys.empty();
if (use_cas_modules) {
if (llvm::all_of(invocation.getFrontendOpts().ModuleCacheKeys,
[&](const auto &entry) {
if (!IsModuleAvailableInCAS(entry.second)) {
std::string warn;
llvm::raw_string_ostream(warn)
<< "Failed to load module '" << entry.first
<< "' from CAS using key: " << entry.second;
AddDiagnostic(eSeverityWarning, warn);
return false;
}
return true;
})) {
// Load from CAS.
invocation.getCASOpts().CASPath = GetCASOptions().CASOpts.CASPath;
invocation.getCASOpts().PluginPath = GetCASOptions().CASOpts.PluginPath;
invocation.getCASOpts().PluginOptions =
GetCASOptions().CASOpts.PluginOptions;
} else {
// Failed to load from CAS.
ClearCASOptions();
invocation.getHeaderSearchOpts().PrebuiltModuleFiles.clear();
invocation.getFrontendOpts().ModuleFiles.clear();
}
} else {
// Clear module cache key and other CAS options to load modules from disk
// directly.
ClearCASOptions();

// Remove non-existing modules in a systematic way.
auto CheckFileExists = [&](const std::string &file) -> bool {
Expand Down Expand Up @@ -3851,12 +3859,27 @@ ThreadSafeASTContext SwiftASTContext::GetASTContext() {
// 2. Create the explicit swift module loader.
if (props.GetUseSwiftExplicitModuleLoader()) {
auto &search_path_opts = GetCompilerInvocation().getSearchPathOptions();
std::unique_ptr<swift::ModuleLoader> esml_up =
swift::ExplicitSwiftModuleLoader::create(
*m_ast_context_up, m_dependency_tracker.get(), loading_mode,
std::unique_ptr<swift::ModuleLoader> esml_up = nullptr;
if (m_cas && m_action_cache) {
if (auto casid = m_cas->parseID(search_path_opts.ExplicitSwiftModuleMapPath)) {
esml_up = swift::ExplicitCASModuleLoader::create(
*m_ast_context_up, *m_cas, *m_action_cache,
m_dependency_tracker.get(), loading_mode,
search_path_opts.ExplicitSwiftModuleMapPath,
search_path_opts.ExplicitSwiftModuleInputs,
/*IgnoreSwiftSourceInfo*/ false);
} else {
// Not CASID, ignore error and treat as a file based moduel map.
llvm::consumeError(casid.takeError());
}
}
if (!esml_up) {
esml_up = swift::ExplicitSwiftModuleLoader::create(
*m_ast_context_up, m_dependency_tracker.get(), loading_mode,
search_path_opts.ExplicitSwiftModuleMapPath,
search_path_opts.ExplicitSwiftModuleInputs,
/*IgnoreSwiftSourceInfo*/ false);
}
if (esml_up) {
m_explicit_swift_module_loader =
static_cast<swift::ExplicitSwiftModuleLoader *>(esml_up.get());
Expand Down
5 changes: 5 additions & 0 deletions lldb/test/API/lang/swift/clangimporter/caching/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,9 @@ SWIFT_SOURCES := main.swift
SWIFT_ENABLE_EXPLICIT_MODULES := YES
SWIFTFLAGS_EXTRAS = -I$(SRCDIR) -I/TEST_DIR -F/FRAMEWORK_DIR -cache-compile-job -cas-path $(BUILDDIR)/cas

all: a.out cas-config

include Makefile.rules

cas-config:
echo "{\"CASPath\": \"$(BUILDDIR)/cas\"}" > .cas-config
25 changes: 6 additions & 19 deletions lldb/test/Shell/Swift/caching.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,15 @@
# RUN: -cache-compile-job -cas-path %t/cas -explicit-module-build \
# RUN: -module-name main -o %t/main

# RUN: %lldb %t/main -s %t/lldb.script 2>&1 | FileCheck %s
# RUN: sed "s|DIR|%/t|g" %t/lldb.script.template > %t/lldb.script
# RUN: %lldb %t/main -s %t/lldb.script 2>&1 | FileCheck %s --check-prefix=CAS-LOAD --check-prefix=CHECK

## Setup CAS and try loading from CAS
# RUN: sed "s|DIR|%/t|g" %t/lldb.script.template > %t/lldb_2.script
# RUN: %lldb %t/main -s %t/lldb_2.script 2>&1 | FileCheck %s --check-prefix=CAS-LOAD --check-prefix=CHECK

## Check fallback to file system.
## Check warning message
# RUN: rm -rf %t/cas
# RUN: %lldb %t/main -s %t/lldb_2.script 2>&1 | FileCheck %s --check-prefix=CAS-FALLBACK --check-prefix=CHECK
# RUN: %lldb %t/main -s %t/lldb.script 2>&1 | FileCheck %s --check-prefix=CAS-FALLBACK --check-prefix=CHECK

# CAS-FALLBACK: operator()() -- module '{{.*}}' cannot be load from CAS using key
# CAS-FALLBACK-SAME: fallback to load from file system
# CAS-FALLBACK: warning: Failed to load module
# CAS-FALLBACK-SAME: from CAS using key:
# CAS-LOAD: ConfigureCASStorage() -- Setup CAS from module list properties with cas path
# CHECK: LogConfiguration() -- Extra clang arguments
# CHECK-COUNT-1: LogConfiguration() -- -triple
Expand All @@ -32,16 +29,6 @@ func test() {
}
test()

//--- lldb.script
# Force loading from interface to simulate no binary module available.
settings set symbols.swift-module-loading-mode prefer-interface
log enable lldb types
b test
run
# Create a SwiftASTContext
expr 1
quit

//--- lldb.script.template
# Force loading from interface to simulate no binary module available.
settings set symbols.swift-module-loading-mode prefer-interface
Expand Down