Add package deduplication, --deduplicatePackages

internal/checker/checker.go

@@ -14871,7 +14871,7 @@ func (c *Checker) resolveExternalModule(location *ast.Node, moduleReference stri
 			return c.getMergedSymbol(sourceFile.Symbol)
 		}
 		if errorNode != nil && moduleNotFoundError != nil && !isSideEffectImport(errorNode) {
-			c.error(errorNode, diagnostics.File_0_is_not_a_module, sourceFile.FileName())
+			c.error(errorNode, diagnostics.File_0_is_not_a_module, resolvedModule.ResolvedFileName)
 		}
 		return nil
 	}

internal/compiler/fileloader.go

@@ -68,7 +68,11 @@ type processedFiles struct {
 	// if file was included using source file and its output is actually part of program
 	// this contains mapping from output to source file
 	outputFileToProjectReferenceSource map[tspath.Path]string
-	finishedProcessing                 bool
+	// Key is a file path. Value is the list of files that redirect to it (same package, different install location)
+	redirectTargetsMap map[tspath.Path][]string
+	// Any paths involved in deduplication, including canonical paths and redirected paths
+	deduplicatedPaths  collections.Set[tspath.Path]
+	finishedProcessing bool
 }
 
 type jsxRuntimeImportSpecifier struct {
@@ -189,6 +193,7 @@ func (p *fileLoader) resolveAutomaticTypeDirectives(containingFileName string) (
 						kind: fileIncludeKindAutomaticTypeDirectiveFile,
 						data: &automaticTypeDirectiveFileData{name, resolved.PackageId},
 					},
+					packageId: resolved.PackageId,
 				})
 			}
 		}
@@ -359,6 +364,7 @@ func (p *fileLoader) resolveTypeReferenceDirectives(t *parseTask) {
 				increaseDepth: resolved.IsExternalLibraryImport,
 				elideOnDepth:  false,
 				includeReason: includeReason,
+				packageId:     resolved.PackageId,
 			}, nil)
 		} else {
 			t.processingDiagnostics = append(t.processingDiagnostics, &processingDiagnostic{
@@ -465,6 +471,7 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(t *parseTask) {
 							synthetic: core.IfElse(importIndex < 0, entry, nil),
 						},
 					},
+					packageId: resolvedModule.PackageId,
 				}, nil)
 			}
 		}

internal/compiler/filesparser.go

@@ -25,6 +25,7 @@ type parseTask struct {
 	startedSubTasks             bool
 	isForAutomaticTypeDirective bool
 	includeReason               *FileIncludeReason
+	packageId                   module.PackageId
 
 	metadata                     ast.SourceFileMetaData
 	resolutionsInFile            module.ModeAwareCache[*module.ResolvedModule]
@@ -164,6 +165,7 @@ type resolvedRef struct {
 	increaseDepth bool
 	elideOnDepth  bool
 	includeReason *FileIncludeReason
+	packageId     module.PackageId
 }
 
 func (t *parseTask) addSubTask(ref resolvedRef, libFile *LibFile) {
@@ -174,6 +176,7 @@ func (t *parseTask) addSubTask(ref resolvedRef, libFile *LibFile) {
 		increaseDepth:      ref.increaseDepth,
 		elideOnDepth:       ref.elideOnDepth,
 		includeReason:      ref.includeReason,
+		packageId:          ref.packageId,
 	}
 	t.subTasks = append(t.subTasks, subTask)
 }
@@ -190,6 +193,7 @@ type parseTaskData struct {
 	mu              sync.Mutex
 	lowestDepth     int
 	startedSubTasks bool
+	packageId       module.PackageId
 }
 
 func (w *filesParser) parse(loader *fileLoader, tasks []*parseTask) {
@@ -220,6 +224,11 @@ func (w *filesParser) start(loader *fileLoader, tasks []*parseTask, depth int) {
 				}
 			}
 
+			// Propagate packageId to data if we have one and data doesn't yet
+			if data.packageId.Name == "" && task.packageId.Name != "" {
+				data.packageId = task.packageId
+			}
+
 			currentDepth := core.IfElse(task.increaseDepth, depth+1, depth)
 			if currentDepth < data.lowestDepth {
 				// If we're seeing this task at a lower depth than before,
@@ -283,6 +292,14 @@ func (w *filesParser) getProcessedFiles(loader *fileLoader) processedFiles {
 	var sourceFilesFoundSearchingNodeModules collections.Set[tspath.Path]
 	libFilesMap := make(map[tspath.Path]*LibFile, libFileCount)
 
+	var redirectTargetsMap map[tspath.Path][]string
+	var deduplicatedPaths collections.Set[tspath.Path]
+	var packageIdToCanonicalPath map[module.PackageId]tspath.Path
+	if !loader.opts.Config.CompilerOptions().DeduplicatePackages.IsFalse() {
+		redirectTargetsMap = make(map[tspath.Path][]string)
+		packageIdToCanonicalPath = make(map[module.PackageId]tspath.Path)
+	}
+
 	var collectFiles func(tasks []*parseTask, seen map[*parseTaskData]string)
 	collectFiles = func(tasks []*parseTask, seen map[*parseTaskData]string) {
 		for _, task := range tasks {
@@ -331,8 +348,23 @@ func (w *filesParser) getProcessedFiles(loader *fileLoader) processedFiles {
 			for _, trace := range task.resolutionsTrace {
 				loader.opts.Host.Trace(trace.Message, trace.Args...)
 			}
-			if subTasks := task.subTasks; len(subTasks) > 0 {
-				collectFiles(subTasks, seen)
+
+			var existingCanonicalPath tspath.Path
+			if packageIdToCanonicalPath != nil && data.packageId.Name != "" {
+				if canonical, exists := packageIdToCanonicalPath[data.packageId]; exists {
+					redirectTargetsMap[canonical] = append(redirectTargetsMap[canonical], task.normalizedFilePath)
+					existingCanonicalPath = canonical
+					deduplicatedPaths.Add(task.path)
+					deduplicatedPaths.Add(canonical)
+				} else {
+					packageIdToCanonicalPath[data.packageId] = task.path
+				}
+			}
+
+			if existingCanonicalPath == "" {
+				if subTasks := task.subTasks; len(subTasks) > 0 {
+					collectFiles(subTasks, seen)
+				}
 			}
 
 			// Exclude automatic type directive tasks from include reason processing,
@@ -350,6 +382,10 @@ func (w *filesParser) getProcessedFiles(loader *fileLoader) processedFiles {
 				continue
 			}
 			file := task.file
+			if existingCanonicalPath != "" {
+				file = filesByPath[existingCanonicalPath]
+			}
+
 			path := task.path
 
 			if len(task.processingDiagnostics) > 0 {
@@ -365,7 +401,7 @@ func (w *filesParser) getProcessedFiles(loader *fileLoader) processedFiles {
 			if task.libFile != nil {
 				libFiles = append(libFiles, file)
 				libFilesMap[path] = task.libFile
-			} else {
+			} else if existingCanonicalPath == "" {
 				files = append(files, file)
 			}
 			filesByPath[path] = file
@@ -424,6 +460,8 @@ func (w *filesParser) getProcessedFiles(loader *fileLoader) processedFiles {
 		missingFiles:                         missingFiles,
 		includeProcessor:                     includeProcessor,
 		outputFileToProjectReferenceSource:   outputFileToProjectReferenceSource,
+		redirectTargetsMap:                   redirectTargetsMap,
+		deduplicatedPaths:                    deduplicatedPaths,
 	}
 }
 

internal/compiler/program.go

@@ -116,9 +116,10 @@ func (p *Program) GetPackageJsonInfo(pkgJsonPath string) *packagejson.InfoCacheE
 	return nil
 }
 
-// GetRedirectTargets implements checker.Program.
+// GetRedirectTargets returns the list of file paths that redirect to the given path.
+// These are files from the same package (same name@version) installed in different locations.
 func (p *Program) GetRedirectTargets(path tspath.Path) []string {
-	return nil // !!! TODO: project references support
+	return p.redirectTargetsMap[path]
 }
 
 // gets the original file that was included in program
@@ -247,6 +248,13 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path, newHost CompilerHos
 	if !canReplaceFileInProgram(oldFile, newFile) {
 		return NewProgram(newOpts), false
 	}
+	// If this file is part of a package redirect group (same package installed in multiple
+	// node_modules locations), we need to rebuild the program because the redirect targets
+	// might need recalculation.
+	if p.deduplicatedPaths.Has(changedFilePath) {
+		// File is either a canonical file or a redirect target; either way, need full rebuild
+		return NewProgram(newOpts), false
+	}
 	// TODO: reverify compiler options when config has changed?
 	result := &Program{
 		opts:                        newOpts,

internal/core/compileroptions.go

@@ -36,6 +36,7 @@ type CompilerOptions struct {
 	Declaration                               Tristate                                  `json:"declaration,omitzero"`
 	DeclarationDir                            string                                    `json:"declarationDir,omitzero"`
 	DeclarationMap                            Tristate                                  `json:"declarationMap,omitzero"`
+	DeduplicatePackages                       Tristate                                  `json:"deduplicatePackages,omitzero"`
 	DisableSizeLimit                          Tristate                                  `json:"disableSizeLimit,omitzero"`
 	DisableSourceOfProjectReferenceRedirect   Tristate                                  `json:"disableSourceOfProjectReferenceRedirect,omitzero"`
 	DisableSolutionSearching                  Tristate                                  `json:"disableSolutionSearching,omitzero"`

internal/diagnostics/extraDiagnosticMessages.json

@@ -86,5 +86,9 @@
     "Option '{0}' requires value to be greater than '{1}'.": {
         "category": "Error",
         "code": 5002
+    },
+    "Deduplicate packages with the same name and version.": {
+        "category": "Message",
+        "code": 100011
     }
 }

internal/fourslash/_scripts/failingTests.txt

@@ -231,7 +231,6 @@ TestContextuallyTypedFunctionExpressionGeneric1
 TestContextualTypingOfGenericCallSignatures2
 TestCrossFileQuickInfoExportedTypeDoesNotUseImportType
 TestDoubleUnderscoreCompletions
-TestDuplicatePackageServices
 TestEditJsdocType
 TestErrorsAfterResolvingVariableDeclOfMergedVariableAndClassDecl
 TestExportDefaultClass

internal/fourslash/_scripts/manualTests.txt

@@ -2,6 +2,7 @@ completionListInClosedFunction05
 completionsAtIncompleteObjectLiteralProperty
 completionsSelfDeclaring1
 completionsWithDeprecatedTag4
+duplicatePackageServices_fileChanges
 navigationBarFunctionPrototype
 navigationBarFunctionPrototype2
 navigationBarFunctionPrototype3
@@ -38,4 +39,4 @@ stringLiteralCompletionsInPositionTypedUsingRest
 importNameCodeFix_uriStyleNodeCoreModules1
 completionsImport_filteredByPackageJson_ambient
 importNameCodeFixDefaultExport7
-importNameCodeFixOptionalImport0
+importNameCodeFixOptionalImport0

internal/fourslash/tests/manual/duplicatePackageServices_fileChanges_test.go

@@ -0,0 +1,68 @@
+package fourslash_test
+
+import (
+	"testing"
+
+	"github.com/microsoft/typescript-go/internal/fourslash"
+	"github.com/microsoft/typescript-go/internal/testutil"
+)
+
+func TestDuplicatePackageServices_fileChanges(t *testing.T) {
+	t.Parallel()
+
+	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
+	const content = `// @noImplicitReferences: true
+// @Filename: /node_modules/a/index.d.ts
+import X from "x";
+export function a(x: X): void;
+// @Filename: /node_modules/a/node_modules/x/index.d.ts
+export default class /*defAX*/X {
+    private x: number;
+}
+// @Filename: /node_modules/a/node_modules/x/package.json
+{ "name": "x", "version": "1.2./*aVersionPatch*/3" }
+// @Filename: /node_modules/b/index.d.ts
+import X from "x";
+export const b: X;
+// @Filename: /node_modules/b/node_modules/x/index.d.ts
+export default class /*defBX*/X {
+    private x: number;
+}
+// @Filename: /node_modules/b/node_modules/x/package.json
+{ "name": "x", "version": "1.2./*bVersionPatch*/3" }
+// @Filename: /src/a.ts
+import { a } from "a";
+import { b } from "b";
+a(/*error*/b);`
+	f, done := fourslash.NewFourslash(t, nil /*capabilities*/, content)
+	defer done()
+
+	f.GoToFile(t, "/src/a.ts")
+	f.VerifyNumberOfErrorsInCurrentFile(t, 0)
+
+	testChangeAndChangeBack := func(versionPatch string, def string) {
+		// Insert "4" after the version patch marker, changing version from 1.2.3 to 1.2.43
+		f.GoToMarker(t, versionPatch)
+		f.Insert(t, "4")
+
+		// Insert a space after the definition marker to trigger a recheck
+		f.GoToMarker(t, def)
+		f.Insert(t, " ")
+
+		// No longer have identical packageId, so we get errors.
+		f.VerifyErrorExistsAfterMarker(t, "error")
+
+		// Undo the changes
+		f.GoToMarker(t, versionPatch)
+		f.DeleteAtCaret(t, 1)
+		f.GoToMarker(t, def)
+		f.DeleteAtCaret(t, 1)
+
+		// Back to being identical.
+		f.GoToFile(t, "/src/a.ts")
+		f.VerifyNumberOfErrorsInCurrentFile(t, 0)
+	}
+
+	testChangeAndChangeBack("aVersionPatch", "defAX")
+	testChangeAndChangeBack("bVersionPatch", "defBX")
+}

internal/module/resolver.go

@@ -986,7 +986,7 @@ func (r *resolutionState) loadModuleFromSpecificNodeModulesDirectory(ext extensi
 			}
 
 			if fromDirectory := r.loadNodeModuleFromDirectoryWorker(ext, candidate, !nodeModulesDirectoryExists, packageInfo); !fromDirectory.shouldContinueSearching() {
-				fromDirectory.packageId = r.getPackageId(packageDirectory, packageInfo)
+				fromDirectory.packageId = r.getPackageId(fromDirectory.path, packageInfo)
 				return fromDirectory
 			}
 		}
@@ -995,12 +995,12 @@ func (r *resolutionState) loadModuleFromSpecificNodeModulesDirectory(ext extensi
 	loader := func(extensions extensions, candidate string, onlyRecordFailures bool) *resolved {
 		if rest != "" || !r.esmMode {
 			if fromFile := r.loadModuleFromFile(extensions, candidate, onlyRecordFailures); !fromFile.shouldContinueSearching() {
-				fromFile.packageId = r.getPackageId(packageDirectory, packageInfo)
+				fromFile.packageId = r.getPackageId(fromFile.path, packageInfo)
 				return fromFile
 			}
 		}
 		if fromDirectory := r.loadNodeModuleFromDirectoryWorker(extensions, candidate, onlyRecordFailures, packageInfo); !fromDirectory.shouldContinueSearching() {
-			fromDirectory.packageId = r.getPackageId(packageDirectory, packageInfo)
+			fromDirectory.packageId = r.getPackageId(fromDirectory.path, packageInfo)
 			return fromDirectory
 		}
 		// !!! this is ported exactly, but checking for null seems wrong?
@@ -1010,7 +1010,7 @@ func (r *resolutionState) loadModuleFromSpecificNodeModulesDirectory(ext extensi
 			// EsmMode disables index lookup in `loadNodeModuleFromDirectoryWorker` generally, however non-relative package resolutions still assume
 			// a default `index.js` entrypoint if no `main` or `exports` are present
 			if indexResult := r.loadModuleFromFile(extensions, tspath.CombinePaths(candidate, "index.js"), onlyRecordFailures); !indexResult.shouldContinueSearching() {
-				indexResult.packageId = r.getPackageId(packageDirectory, packageInfo)
+				indexResult.packageId = r.getPackageId(indexResult.path, packageInfo)
 				return indexResult
 			}
 		}

internal/tsoptions/declscompiler.go

@@ -185,6 +185,14 @@ var commonOptionsWithBuild = []*CommandLineOption{
 		DefaultValueDescription:  false,
 		// Not setting affectsSemanticDiagnostics or affectsBuildInfo because we dont want all diagnostics to go away, its handled in builder
 	},
+	{
+		Name:                    "deduplicatePackages",
+		Kind:                    CommandLineOptionTypeBoolean,
+		Category:                diagnostics.Type_Checking,
+		Description:             diagnostics.Deduplicate_packages_with_the_same_name_and_version,
+		DefaultValueDescription: true,
+		AffectsProgramStructure: true,
+	},
 	{
 		Name:                     "noEmit",
 		Kind:                     CommandLineOptionTypeBoolean,

internal/tsoptions/parsinghelpers.go

@@ -217,6 +217,8 @@ func parseCompilerOptions(key string, value any, allOptions *core.CompilerOption
 		allOptions.Composite = ParseTristate(value)
 	case "declarationDir":
 		allOptions.DeclarationDir = ParseString(value)
+	case "deduplicatePackages":
+		allOptions.DeduplicatePackages = ParseTristate(value)
 	case "diagnostics":
 		allOptions.Diagnostics = ParseTristate(value)
 	case "disableSizeLimit":