Skip to content

feat: add configuration cache compatibility #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ffluk3 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add configuration cache compatibility #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 71 additions & 56 deletions ...duledetector/src/main/kotlin/com/dropbox/affectedmoduledetector/AffectedModuleDetector.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@ import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.logging.Logger
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters
Expand Down Expand Up @@ -155,28 +158,20 @@ abstract class AffectedModuleDetector(protected val logger: Logger?) {
"extension added."
}

val distDir = if (config.logFolder != null) {
val distDir = File(config.logFolder!!)
if (!distDir.exists()) {
distDir.mkdirs()
}
distDir
val logFile = if (config.logFolder != null) {
File(config.logFolder!!).resolve(config.logFilename)
} else {
rootProject.rootDir
}

val outputFile = distDir.resolve(config.logFilename).also {
it.writeText("")
rootProject.rootDir.resolve(config.logFilename)
}
val logger = FileLogger(outputFile)

val enabled = isProjectEnabled(rootProject)
if (!enabled) {
val provider =
setupWithParams(rootProject) { spec ->
val params = spec.parameters
params.acceptAll = true
params.log = logger
params.acceptAll.set(true)
params.logFile.set(logFile)
params.workingDir.set(rootProject.projectDir)
}
instance.wrapped = provider
return
Expand All @@ -195,14 +190,11 @@ abstract class AffectedModuleDetector(protected val logger: Logger?) {
}
}

val modules =
getModulesProperty(
rootProject
)
val modules = getModulesProperty(rootProject)

val gitClient = GitClientImpl(
rootProject.projectDir,
logger,
null,
commitShaProviderConfiguration = CommitShaProviderConfiguration(
type = config.compareFrom,
specifiedBranch = config.specifiedBranch,
Expand All @@ -213,23 +205,22 @@ abstract class AffectedModuleDetector(protected val logger: Logger?) {
ignoredFiles = config.ignoredFiles
)

logger.lifecycle("projects evaluated")
val projectGraph = ProjectGraph(rootProject, logger.toLogger())
val dependencyTracker = DependencyTracker(rootProject, logger.toLogger())
val fileLogger = FileLogger(logFile)
val projectGraph = ProjectGraph(rootProject, fileLogger.toLogger())
val dependencyTracker = DependencyTracker(rootProject, fileLogger.toLogger())
val provider = setupWithParams(rootProject) { spec ->
val parameters = spec.parameters
parameters.acceptAll = false
parameters.projectGraph = projectGraph
parameters.dependencyTracker = dependencyTracker
parameters.log = logger
parameters.ignoreUnknownProjects = true
parameters.projectSubset = subset
parameters.modules = modules
parameters.config = config
parameters.gitChangedFilesProvider = gitClient.findChangedFiles(rootProject)
parameters.gitRoot.set(gitClient.getGitRoot())
parameters.acceptAll.set(false)
parameters.projectGraph.set(projectGraph)
parameters.dependencyTracker.set(dependencyTracker)
parameters.logFile.set(logFile)
parameters.ignoreUnknownProjects.set(true)
parameters.projectSubset.set(subset)
if (modules != null) parameters.modules.set(modules.toList())
parameters.config.set(config)
parameters.gitChangedFilesProvider.set(gitClient.findChangedFiles(rootProject))
parameters.workingDir.set(rootProject.projectDir)
}
logger.info("Using real detector with $subset")
instance.wrapped = provider
}

Expand Down Expand Up @@ -288,10 +279,13 @@ abstract class AffectedModuleDetector(protected val logger: Logger?) {
@Throws(GradleException::class)
@JvmStatic
fun configureTaskGuard(task: Task) {
// Capture only serializable values (String path) rather than live Task/Project
// references, which are not configuration-cache compatible.
val projectPath = task.project.path
val detectorExtension = task.project.rootProject.extensions.getByName(ROOT_PROP_NAME)
as AffectedModuleDetectorWrapper
task.onlyIf {
getOrThrow(
task.project
).shouldInclude(task.project.projectPath)
detectorExtension.shouldInclude(ProjectPath(projectPath))
}
}

Expand Down Expand Up @@ -402,36 +396,57 @@ class AffectedModuleDetectorWrapper : AffectedModuleDetector(logger = null) {
abstract class AffectedModuleDetectorLoader :
BuildService<AffectedModuleDetectorLoader.Parameters> {
interface Parameters : BuildServiceParameters {
var acceptAll: Boolean
var projectGraph: ProjectGraph
var dependencyTracker: DependencyTracker
var log: FileLogger
var ignoreUnknownProjects: Boolean
var projectSubset: ProjectSubset
var modules: Set<String>?
var gitChangedFilesProvider: Provider<List<String>>
var config: AffectedModuleConfiguration
val gitRoot: DirectoryProperty
val acceptAll: Property<Boolean>
val projectGraph: Property<ProjectGraph>
val dependencyTracker: Property<DependencyTracker>
val logFile: RegularFileProperty
val ignoreUnknownProjects: Property<Boolean>
val projectSubset: Property<ProjectSubset>
val modules: ListProperty<String>
val gitChangedFilesProvider: Property<Provider<List<String>>>
val config: Property<AffectedModuleConfiguration>
// workingDir is used to locate the git root inside the service, avoiding a
// filesystem walk at configuration time.
val workingDir: DirectoryProperty
}

val detector: AffectedModuleDetector by lazy {
val logger = parameters.log.toLogger()
if (parameters.acceptAll) {
val logFile = parameters.logFile.orNull?.asFile
if (logFile != null) {
logFile.parentFile?.mkdirs()
if (!logFile.exists()) logFile.createNewFile()
}
val logger = logFile?.let { FileLogger(it).toLogger() }

if (parameters.acceptAll.get()) {
AcceptAll(logger)
} else {
val workingDir = parameters.workingDir.get().asFile
val gitRoot = findGitDirInParentFilepath(workingDir) ?: workingDir
AffectedModuleDetectorImpl(
projectGraph = parameters.projectGraph,
dependencyTracker = parameters.dependencyTracker,
projectGraph = parameters.projectGraph.get(),
dependencyTracker = parameters.dependencyTracker.get(),
logger = logger,
ignoreUnknownProjects = parameters.ignoreUnknownProjects,
projectSubset = parameters.projectSubset,
modules = parameters.modules,
config = parameters.config,
changedFilesProvider = parameters.gitChangedFilesProvider,
gitRoot = parameters.gitRoot.get().asFile
ignoreUnknownProjects = parameters.ignoreUnknownProjects.get(),
projectSubset = parameters.projectSubset.get(),
modules = parameters.modules.orNull?.toSet()?.takeIf { it.isNotEmpty() },
config = parameters.config.get(),
changedFilesProvider = parameters.gitChangedFilesProvider.get(),
gitRoot = gitRoot
)
}
}

private fun findGitDirInParentFilepath(filepath: File): File? {
var curDirectory: File = filepath
while (curDirectory.path != "/") {
if (File("$curDirectory/.git").exists()) {
return curDirectory
}
curDirectory = curDirectory.parentFile
}
return null
}
}

/**
Expand Down
27 changes: 8 additions & 19 deletions ...tector/src/main/kotlin/com/dropbox/affectedmoduledetector/AffectedModuleDetectorPlugin.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.testing.Test
import org.gradle.util.GradleVersion
import org.jetbrains.annotations.VisibleForTesting

/**
Expand Down Expand Up @@ -90,7 +89,6 @@ class AffectedModuleDetectorPlugin : Plugin<Project> {
rootProject.tasks.register(taskType.commandByImpact) { task ->
task.group = CUSTOM_TASK_GROUP_NAME
task.description = taskType.taskDescription
disableConfigCache(task)

rootProject.subprojects { project ->
pluginIds.forEach { pluginId ->
Expand Down Expand Up @@ -131,7 +129,6 @@ class AffectedModuleDetectorPlugin : Plugin<Project> {
rootProject.tasks.register(taskType.commandByImpact) { task ->
task.group = groupName
task.description = taskType.taskDescription
disableConfigCache(task)

rootProject.subprojects { project ->
pluginIds.forEach { pluginId ->
Expand All @@ -147,13 +144,6 @@ class AffectedModuleDetectorPlugin : Plugin<Project> {
}
}

@Suppress("UnstableApiUsage")
private fun disableConfigCache(task: Task) {
if (GradleVersion.current() >= GradleVersion.version("7.4")) {
task.notCompatibleWithConfigurationCache("AMD requires knowledge of what has changed in the file system so we can not cache those values (https://github.com/dropbox/AffectedModuleDetector/issues/150)")
}
}

private fun withPlugin(
pluginId: String,
task: Task,
Expand All @@ -169,16 +159,15 @@ class AffectedModuleDetectorPlugin : Plugin<Project> {
project.pluginManager.withPlugin(pluginId) {
getAffectedPath(testType, project)?.let { path ->
val pathOrNull = project.tasks.findByPath(path)
val onlyIf = when {
pathOrNull == null -> false
!AffectedModuleDetector.isProjectEnabled(pathOrNull.project) -> true
else -> AffectedModuleDetector.isProjectAffected(pathOrNull.project)
}
if (pathOrNull == null) return@withPlugin

if (onlyIf && AffectedModuleDetector.isProjectProvided(project) && !isExcludedModule(config, path)) {
task.dependsOn(path)
}
pathOrNull?.onlyIf { onlyIf }
val isProvided = AffectedModuleDetector.isProjectProvided(project)
if (!isProvided || isExcludedModule(config, path)) return@withPlugin

task.dependsOn(path)
// Defer the affected/not-affected decision to execution time via configureTaskGuard
// so neither the result nor any Project/Task reference is baked into the cache.
AffectedModuleDetector.configureTaskGuard(pathOrNull)
}
}
}
Expand Down
45 changes: 27 additions & 18 deletions ...tedmoduledetector/src/main/kotlin/com/dropbox/affectedmoduledetector/DependencyTracker.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,26 +30,35 @@ import java.io.Serializable
*/
class DependencyTracker(rootProject: Project, logger: Logger?) : Serializable {

private val dependentList: Map<ProjectPath, Set<ProjectPath>> by lazy {
val result = mutableMapOf<ProjectPath, MutableSet<ProjectPath>>()
rootProject.subprojects.forEach { project ->
logger?.info("checking ${project.path} for dependencies")
project.configurations.forEach { config ->
logger?.info("checking config ${project.path}/$config for dependencies")
config
.dependencies
.filterIsInstance<ProjectDependency>()
.forEach {
logger?.info(
"there is a dependency from ${project.projectPath} to " +
it.path,
)
result.getOrPut(ProjectPath(it.path)) { mutableSetOf() }
.add(project.projectPath)
}
// Eagerly computed at construction so no live Project reference is retained in a
// serializable field (required for Gradle configuration cache compatibility).
private val dependentList: Map<ProjectPath, Set<ProjectPath>> = buildDependentList(rootProject, logger)

companion object {
private fun buildDependentList(
rootProject: Project,
logger: Logger?
): Map<ProjectPath, Set<ProjectPath>> {
val result = mutableMapOf<ProjectPath, MutableSet<ProjectPath>>()
rootProject.subprojects.forEach { project ->
logger?.info("checking ${project.path} for dependencies")
project.configurations.forEach { config ->
logger?.info("checking config ${project.path}/$config for dependencies")
config
.dependencies
.filterIsInstance<ProjectDependency>()
.forEach {
logger?.info(
"there is a dependency from ${project.projectPath} to " +
it.path,
)
result.getOrPut(ProjectPath(it.path)) { mutableSetOf() }
.add(project.projectPath)
}
}
}
return result
}
result
}

fun findAllDependents(projectPath: ProjectPath, logger: Logger? = null): Set<ProjectPath> {
Expand Down
24 changes: 14 additions & 10 deletions affectedmoduledetector/src/main/kotlin/com/dropbox/affectedmoduledetector/GitClient.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ import com.dropbox.affectedmoduledetector.util.toOsSpecificLineEnding
import com.dropbox.affectedmoduledetector.util.toOsSpecificPath
import org.gradle.api.Project
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.logging.Logger
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.provider.SetProperty
import org.gradle.api.provider.ValueSource
Expand Down Expand Up @@ -91,9 +93,9 @@ internal class GitClientImpl(
project: Project,
): Provider<List<String>> {
return project.providers.of(GitChangedFilesSource::class.java) {
it.parameters.commitShaProvider = commitShaProviderConfiguration
it.parameters.commitShaProvider.set(commitShaProviderConfiguration)
it.parameters.workingDir.set(workingDir)
it.parameters.logger = logger
it.parameters.logFile.set(logger?.file)
it.parameters.ignoredFiles.set(ignoredFiles)
}
}
Expand Down Expand Up @@ -176,9 +178,9 @@ private class RealCommandRunner(
internal abstract class GitChangedFilesSource :
ValueSource<List<String>, GitChangedFilesSource.Parameters> {
interface Parameters : ValueSourceParameters {
var commitShaProvider: CommitShaProviderConfiguration
val commitShaProvider: Property<CommitShaProviderConfiguration>
val workingDir: DirectoryProperty
var logger: FileLogger?
val logFile: RegularFileProperty
val ignoredFiles: SetProperty<String>
}

Expand All @@ -189,17 +191,18 @@ internal abstract class GitChangedFilesSource :
private val commandRunner: GitClient.CommandRunner by lazy {
RealCommandRunner(
workingDir = gitRoot ?: parameters.workingDir.get().asFile,
logger = null
logger = parameters.logFile.orNull?.asFile?.let { FileLogger(it).toLogger() }
)
}

override fun obtain(): List<String> {
val top = parameters.commitShaProvider.top
val commitShaProviderConfig = parameters.commitShaProvider.get()
val top = commitShaProviderConfig.top
val sha = getSha()

// use this if we don't want local changes
val changedFiles = commandRunner.executeAndParse(
if (parameters.commitShaProvider.includeUncommitted) {
if (commitShaProviderConfig.includeUncommitted) {
"$CHANGED_FILES_CMD_PREFIX $sha"
} else {
"$CHANGED_FILES_CMD_PREFIX $top..$sha"
Expand Down Expand Up @@ -227,9 +230,10 @@ internal abstract class GitChangedFilesSource :
}

private fun getSha(): Sha {
val specifiedBranch = parameters.commitShaProvider.specifiedBranch
val specifiedSha = parameters.commitShaProvider.specifiedSha
val type = when (parameters.commitShaProvider.type) {
val config = parameters.commitShaProvider.get()
val specifiedBranch = config.specifiedBranch
val specifiedSha = config.specifiedSha
val type = when (config.type) {
"PreviousCommit" -> PreviousCommit()
"ForkCommit" -> ForkCommit()
"SpecifiedBranchCommit" -> {
Expand Down
Loading