diff --git a/build.sbt b/build.sbt index 32095cb..4b2d54e 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import org.scalajs.linker.interface.Report import scala.sys.process._ -ThisBuild / scalaVersion := "2.13.10" +ThisBuild / scalaVersion := "2.13.17" ThisBuild / description := "Salesforce Apex outline parser with type definitions" ThisBuild / organization := "io.github.apex-dev-tools" ThisBuild / organizationHomepage := Some(url("https://github.com/apex-dev-tools/outline-parser")) @@ -18,16 +18,17 @@ ThisBuild / developers := List( url("https://github.com/apex-dev-tools") ) ) -ThisBuild / versionScheme := Some("strict") -ThisBuild / sonatypeCredentialHost := "s01.oss.sonatype.org" -ThisBuild / sonatypeRepository := "https://s01.oss.sonatype.org/service/local" +ThisBuild / versionScheme := Some("strict") ThisBuild / resolvers += Resolver.mavenLocal -ThisBuild / resolvers ++= Resolver.sonatypeOssRepos("releases") -ThisBuild / resolvers ++= Resolver.sonatypeOssRepos("snapshots") +ThisBuild / resolvers += Resolver.sonatypeCentralSnapshots -lazy val build = taskKey[File]("Build artifacts") -lazy val pack = inputKey[Unit]("Publish specific local version") -lazy val Dev = config("dev") extend Compile +// Java 17 development with Java 8 runtime compatibility +ThisBuild / javacOptions ++= Seq("-source", "8", "-target", "8") + +lazy val build = taskKey[File]("Build artifacts") +lazy val pack = inputKey[Unit]("Publish specific local version") +lazy val npmInstall = taskKey[Unit]("Install Node modules for Scala.js tasks") +lazy val Dev = config("dev") extend Compile // Don't publish root publish / skip := true @@ -43,7 +44,7 @@ lazy val parser = crossProject(JSPlatform, JVMPlatform) scalacOptions += "-deprecation", libraryDependencies ++= Seq( "org.scalatest" %%% "scalatest" % "3.2.9" % Test, - "io.github.apex-dev-tools" %%% "apex-ls" % "4.3.1" % Test + "io.github.apex-dev-tools" %%% "apex-ls" % "6.0.2" % Test ) ) .jvmSettings( @@ -55,10 +56,14 @@ lazy val parser = crossProject(JSPlatform, JVMPlatform) ) ) .jsSettings( - build := buildJs(Compile / fullLinkJS).value, - Dev / build := buildJs(Compile / fastLinkJS).value, + build := buildJs(Compile / fullLinkJS).value, + Dev / build := buildJs(Compile / fastLinkJS).value, + Test / parallelExecution := false, + npmInstall := syncNodeModules.value, + Test / test := (Test / test).dependsOn(npmInstall).value, + Test / testOnly := (Test / testOnly).dependsOn(npmInstall).evaluated, + Test / testQuick := (Test / testQuick).dependsOn(npmInstall).evaluated, libraryDependencies ++= Seq("net.exoego" %%% "scala-js-nodejs-v14" % "0.12.0"), - Test / parallelExecution := false, scalaJSUseMainModuleInitializer := false, scalaJSLinkerConfig ~= { _.withModuleKind(ModuleKind.CommonJSModule) @@ -82,34 +87,45 @@ lazy val buildJVM = Def.task { } def buildJs(jsTask: TaskKey[Attributed[Report]]): Def.Initialize[Task[File]] = Def.task { - def exec: (String, File) => Unit = run(streams.value.log)(_, _) - // Depends on scalaJS fast/full linker output - val t = jsTask.value + jsTask.value val targetDir = crossTarget.value val targetFile = (jsTask / scalaJSLinkerOutputDirectory).value / "main.js" - val npmDir = baseDirectory.value / "npm" - val files: Map[File, File] = Map( - // Update target with NPM modules (for testing) - npmDir / "package.json" -> targetDir / "package.json" - ) + syncNodeModules.value - IO.copy(files, CopyOptions().withOverwrite(true)) + targetFile +} - // Install modules in NPM - exec("npm ci", npmDir) +def syncNodeModules: Def.Initialize[Task[Unit]] = Def.task { + val log = streams.value.log + val npmDir = baseDirectory.value / "npm" + val targetDir = crossTarget.value + val lockFile = npmDir / "package-lock.json" + val nodeDir = npmDir / "node_modules" + val exec = run(log)(_, _) + val needInstall = + !nodeDir.exists() || (lockFile.exists() && lockFile.lastModified() > nodeDir.lastModified()) + + if (needInstall) { + exec("npm ci", npmDir) + } - // Update target with NPM modules (for testing) - IO.delete(targetDir / "node_modules") - IO.copyDirectory( - npmDir / "node_modules", - targetDir / "node_modules", - CopyOptions().withOverwrite(true) - ) + val packageJson = npmDir / "package.json" + if (packageJson.exists()) { + IO.copyFile(packageJson, targetDir / "package.json") + } + if (lockFile.exists()) { + IO.copyFile(lockFile, targetDir / "package-lock.json") + } - targetFile + IO.delete(targetDir / "node_modules") + if (nodeDir.exists()) { + IO.copyDirectory(nodeDir, targetDir / "node_modules", CopyOptions().withOverwrite(true)) + } else { + log.warn("npm node_modules directory not found after installation") + } } // Command to do a local release under a specific version diff --git a/js/npm/package-lock.json b/js/npm/package-lock.json index 0affdfc..66c58ab 100644 --- a/js/npm/package-lock.json +++ b/js/npm/package-lock.json @@ -6,16 +6,17 @@ "": { "name": "@apexdevtools/outline-parser", "dependencies": { - "@apexdevtools/apex-parser": "^3.3.0" + "@apexdevtools/apex-parser": "^4.3.1" }, "engines": { "node": ">=14.0.0" } }, "node_modules/@apexdevtools/apex-parser": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/@apexdevtools/apex-parser/-/apex-parser-3.3.0.tgz", - "integrity": "sha512-RK36FBytyIlVUWlH9RXadxEL+aZ9vNG5cEuLFWBuwLAsVQA3xU35Uw4CYDEDSoTjbOHcI8PXcEqTC5i3E/tADA==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/@apexdevtools/apex-parser/-/apex-parser-4.4.1.tgz", + "integrity": "sha512-tLHQ8DkI7/aoL9nOax+Xb3OEXk8IK1mTIpcCBaBJ3kk0Mhy4ik9jfQVAoSxjbWo8aLrjz2E4jnjmSU1iZlEt+Q==", + "license": "BSD-3-Clause", "dependencies": { "antlr4ts": "0.5.0-alpha.4", "node-dir": "^0.1.17" @@ -73,9 +74,9 @@ }, "dependencies": { "@apexdevtools/apex-parser": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/@apexdevtools/apex-parser/-/apex-parser-3.3.0.tgz", - "integrity": "sha512-RK36FBytyIlVUWlH9RXadxEL+aZ9vNG5cEuLFWBuwLAsVQA3xU35Uw4CYDEDSoTjbOHcI8PXcEqTC5i3E/tADA==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/@apexdevtools/apex-parser/-/apex-parser-4.4.1.tgz", + "integrity": "sha512-tLHQ8DkI7/aoL9nOax+Xb3OEXk8IK1mTIpcCBaBJ3kk0Mhy4ik9jfQVAoSxjbWo8aLrjz2E4jnjmSU1iZlEt+Q==", "requires": { "antlr4ts": "0.5.0-alpha.4", "node-dir": "^0.1.17" diff --git a/js/npm/package.json b/js/npm/package.json index bedd405..d511f08 100644 --- a/js/npm/package.json +++ b/js/npm/package.json @@ -1,7 +1,7 @@ { "name": "@apexdevtools/outline-parser", "dependencies": { - "@apexdevtools/apex-parser": "^3.3.0" + "@apexdevtools/apex-parser": "^4.3.1" }, "engines": { "node": ">=14.0.0" diff --git a/js/src/test/scala/com/financialforce/oparser/testutil/AntlrOps.scala b/js/src/test/scala/com/financialforce/oparser/testutil/AntlrOps.scala index c96a78a..cdaad8c 100644 --- a/js/src/test/scala/com/financialforce/oparser/testutil/AntlrOps.scala +++ b/js/src/test/scala/com/financialforce/oparser/testutil/AntlrOps.scala @@ -30,13 +30,20 @@ object AntlrOps { implicit class ContextOps[X <: ParserRuleContext](context: X) { def location: Location = { + // ANTLR uses [start, end) convention with exclusive end positions + // We don't calculate byte offsets here as that would be too expensive + // The comparison code will ignore byte offsets for ANTLR-generated locations + + // Handle UndefOr[Token] in ScalaJS - context.stop might be undefined + val stopToken = context.stop.toOption.getOrElse(context.start) + Location( context.start.line, context.start.charPositionInLine, - context.start.startIndex, - context.stop.line, - context.stop.charPositionInLine + context.stop.text.length, - context.stop.stopIndex + 0, // byte offset not calculated + stopToken.line, + stopToken.charPositionInLine + stopToken.text.length, + 0 // byte offset not calculated ) } } diff --git a/jvm/src/test/scala/com/financialforce/oparser/testutil/AntlrOps.scala b/jvm/src/test/scala/com/financialforce/oparser/testutil/AntlrOps.scala index 27c9ff2..bb8a53c 100644 --- a/jvm/src/test/scala/com/financialforce/oparser/testutil/AntlrOps.scala +++ b/jvm/src/test/scala/com/financialforce/oparser/testutil/AntlrOps.scala @@ -25,13 +25,16 @@ object AntlrOps { implicit class ContextOps(context: ParserRuleContext) { def location: Location = { + // ANTLR uses [start, end) convention with exclusive end positions + // We don't calculate byte offsets here as that would be too expensive + // The comparison code will ignore byte offsets for ANTLR-generated locations Location( context.start.getLine, context.start.getCharPositionInLine, - context.start.getStartIndex, + 0, // byte offset not calculated context.stop.getLine, context.stop.getCharPositionInLine + context.stop.getText.length, - context.stop.getStopIndex + 0 // byte offset not calculated ) } } diff --git a/project/build.properties b/project/build.properties index 9a19778..6e4d63e 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.8.0 +sbt.version = 1.12.8 diff --git a/project/plugins.sbt b/project/plugins.sbt index 27b393d..9678579 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.12.0") -addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.2.0") -addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.11") -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.20.2") +addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.3.2") +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.11.2") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.6") diff --git a/shared/src/main/scala/com/financialforce/oparser/Buffer.scala b/shared/src/main/scala/com/financialforce/oparser/Buffer.scala index 3d3d7a7..3ff28e6 100644 --- a/shared/src/main/scala/com/financialforce/oparser/Buffer.scala +++ b/shared/src/main/scala/com/financialforce/oparser/Buffer.scala @@ -7,15 +7,16 @@ import com.financialforce.types.base.Location class Buffer(backing: String) { - private var startLine = 0 - private var startLineOffset = 0 - private var endLine = 0 - private var endLineOffset = 0 - private var startCharOffset = 0 - private var endCharOffset = 0 - private var startByteOffset = 0 - private var endByteOffset = 0 - private var capturing = false + private var startLine = 0 + private var startLineOffset = 0 + private var endLine = 0 + private var endLineOffset = 0 + private var startCharOffset = 0 + private var endCharOffset = 0 + private var startByteOffset = 0 + private var endByteOffset = 0 + private var lastCharByteLength = 1 + private var capturing = false private val emptyString = "" @@ -23,7 +24,14 @@ class Buffer(backing: String) { ( if (capturing) backing.substring(startCharOffset, endCharOffset + 1) else emptyString, - Location(startLine, startLineOffset, startByteOffset, endLine, endLineOffset, endByteOffset) + Location( + startLine, + startLineOffset, + startByteOffset, + endLine, + endLineOffset + 1, + endByteOffset + lastCharByteLength + ) ) } @@ -36,10 +44,18 @@ class Buffer(backing: String) { endLineOffset = 0 startByteOffset = 0 endByteOffset = 0 + lastCharByteLength = 1 capturing = false } - def append(byteOffset: Int, charOffset: Int, line: Int, lineOffset: Int): Unit = { + def append( + byteOffset: Int, + charOffset: Int, + line: Int, + lineOffset: Int, + charByteLength: Int, + charCount: Int = 1 + ): Unit = { if (!capturing) { capturing = true startByteOffset = byteOffset @@ -48,9 +64,10 @@ class Buffer(backing: String) { startLineOffset = lineOffset } endByteOffset = byteOffset - endCharOffset = charOffset + endCharOffset = charOffset + charCount - 1 // For surrogate pairs, this will be charOffset + 1 endLine = line endLineOffset = lineOffset + lastCharByteLength = charByteLength } } diff --git a/shared/src/main/scala/com/financialforce/oparser/OutlineParser.scala b/shared/src/main/scala/com/financialforce/oparser/OutlineParser.scala index 07fce52..0cc3566 100644 --- a/shared/src/main/scala/com/financialforce/oparser/OutlineParser.scala +++ b/shared/src/main/scala/com/financialforce/oparser/OutlineParser.scala @@ -152,6 +152,7 @@ final class OutlineParser[TypeDecl <: IMutableTypeDeclaration, Ctx]( val startPosition = tokens.head.location.startPosition tokens.clear() consumeClassBody(classTypeDeclaration) + consumeTrailingNewline() classTypeDeclaration.setLocation( Location(startPosition, Position(line, lineOffset, byteOffset)) ) @@ -220,8 +221,10 @@ final class OutlineParser[TypeDecl <: IMutableTypeDeclaration, Ctx]( if (classMembers.length == 1 && classMembers.head.isInstanceOf[PropertyDeclaration]) classMembers.head.asInstanceOf[PropertyDeclaration].propertyBlocks = consumePropertyDeclaration() - else + else { consumeBlock() + consumeTrailingNewline() + } classMembers.foreach(m => m.setLocations(startPosition, startBlockPosition, Position(line, lineOffset, byteOffset)) ) @@ -392,6 +395,7 @@ final class OutlineParser[TypeDecl <: IMutableTypeDeclaration, Ctx]( val startPosition = tokens.head.location.startPosition tokens.clear() consumeInterfaceBody(interfaceTypeDeclaration) + consumeTrailingNewline() interfaceTypeDeclaration.setLocation( Location(startPosition, Position(line, lineOffset, byteOffset)) ) @@ -453,6 +457,7 @@ final class OutlineParser[TypeDecl <: IMutableTypeDeclaration, Ctx]( val startPosition = tokens.head.location.startPosition tokens.clear() consumeEnumBody(enumTypeDeclaration) + consumeTrailingNewline() enumTypeDeclaration.setLocation( Location(startPosition, Position(line, lineOffset, byteOffset)) ) @@ -498,7 +503,7 @@ final class OutlineParser[TypeDecl <: IMutableTypeDeclaration, Ctx]( finished = false charOffset = 0 byteOffset = 0 - lineOffset = 1 + lineOffset = 0 currentChar = contents(charOffset) line = 1 buffer.clear() @@ -512,15 +517,29 @@ final class OutlineParser[TypeDecl <: IMutableTypeDeclaration, Ctx]( private val byteBuffer = new mutable.StringBuilder(4) private def consumeCharacter(capture: Boolean = true): Unit = { - if (capture) - buffer.append(byteOffset, charOffset, line, lineOffset) + if (charOffset + 1 == length) { + // Consuming the last character - increment offsets to position one-past-the-end + val charByteLength = if (currentChar < 128) { + 1 + } else { + byteBuffer.setLength(0) + byteBuffer.append(currentChar) + byteBuffer.toString().getBytes(StandardCharsets.UTF_8).length + } + + if (capture) + buffer.append(byteOffset, charOffset, line, lineOffset, charByteLength) - if (charOffset + 1 == length) + charOffset += 1 + byteOffset += charByteLength + lineOffset += 1 finished = true - else { + } else { currentChar = contents(charOffset + 1) if (currentChar < 128) { // Simple ASCII + if (capture) + buffer.append(byteOffset, charOffset, line, lineOffset, 1) charOffset += 1 byteOffset += 1 lineOffset += 1 @@ -530,23 +549,36 @@ final class OutlineParser[TypeDecl <: IMutableTypeDeclaration, Ctx]( Character.isHighSurrogate(currentChar) && charOffset + 2 != length && Character .isLowSurrogate(contents(charOffset + 2)) ) { - // UTF-16 Surrogate pair + // UTF-16 Surrogate pair (2 UTF-16 chars, but 1 Unicode codepoint) byteBuffer.append(currentChar) currentChar = contents(charOffset + 2) byteBuffer.append(currentChar) + val charByteLength = byteBuffer.toString().getBytes(StandardCharsets.UTF_8).length + if (capture) + buffer.append(byteOffset, charOffset, line, lineOffset, charByteLength, charCount = 2) charOffset += 2 + byteOffset += charByteLength lineOffset += 1 } else { // UTF-16 byteBuffer.append(currentChar) + val charByteLength = byteBuffer.toString().getBytes(StandardCharsets.UTF_8).length + if (capture) + buffer.append(byteOffset, charOffset, line, lineOffset, charByteLength) charOffset += 1 + byteOffset += charByteLength lineOffset += 1 } - byteOffset += byteBuffer.toString().getBytes(StandardCharsets.UTF_8).length } } } + private def consumeTrailingNewline(): Unit = { + // Consume trailing newline to match ANTLR's inclusive behavior + if (!atEnd() && (currentChar == Tokens.Newline || currentChar == Tokens.CarriageReturn)) + consumeNewline() + } + private def isAtComment: Boolean = { currentChar == Tokens.ForwardSlash && (peekMatch(Tokens.Asterisk) || peekMatch(Tokens.ForwardSlash)) @@ -620,12 +652,12 @@ final class OutlineParser[TypeDecl <: IMutableTypeDeclaration, Ctx]( consumeCharacter(false) consumeCharacter(false) line += 1 - lineOffset = 1 + lineOffset = 0 true } else if (currentChar == Tokens.Newline) { consumeCharacter(false) line += 1 - lineOffset = 1 + lineOffset = 0 true } else { consumeCharacter(false) diff --git a/shared/src/main/scala/com/financialforce/oparser/Types.scala b/shared/src/main/scala/com/financialforce/oparser/Types.scala index d011c86..6d32bf9 100644 --- a/shared/src/main/scala/com/financialforce/oparser/Types.scala +++ b/shared/src/main/scala/com/financialforce/oparser/Types.scala @@ -55,13 +55,7 @@ object FormalParameter { typeRef: TypeRef, id: LocatableIdToken ): FormalParameter = { - new FormalParameter( - Annotation.intern(annotations), - Modifier.intern(modifiers), - typeRef, - id.name, - id.location - ) + new FormalParameter(annotations, modifiers, typeRef, id.name, id.location) } } @@ -176,12 +170,7 @@ object ConstructorDeclaration { qName: QualifiedName, formalParameterList: ArraySeq[FormalParameter] ): ConstructorDeclaration = { - new ConstructorDeclaration( - Annotation.intern(annotations), - Modifier.intern(modifiers), - qName, - formalParameterList - ) + new ConstructorDeclaration(annotations, modifiers, qName, formalParameterList) } } @@ -205,13 +194,7 @@ object MethodDeclaration { id: LocatableIdToken, formalParameterList: ArraySeq[FormalParameter] ): MethodDeclaration = { - new MethodDeclaration( - Annotation.intern(annotations), - Modifier.intern(modifiers), - typeRef, - id, - formalParameterList - ) + new MethodDeclaration(annotations, modifiers, typeRef, id, formalParameterList) } } @@ -235,13 +218,7 @@ object PropertyDeclaration { propertyBlocks: Array[PropertyBlock], id: LocatableIdToken ): PropertyDeclaration = { - new PropertyDeclaration( - Annotation.intern(annotations), - Modifier.intern(modifiers), - typeRef, - propertyBlocks, - id - ) + new PropertyDeclaration(annotations, modifiers, typeRef, propertyBlocks, id) } } @@ -263,7 +240,7 @@ object FieldDeclaration { typeRef: TypeRef, id: LocatableIdToken ): FieldDeclaration = { - new FieldDeclaration(Annotation.intern(annotations), Modifier.intern(modifiers), typeRef, id) + new FieldDeclaration(annotations, modifiers, typeRef, id) } } @@ -402,7 +379,7 @@ object Parse { def parseEnumMember(etd: IMutableTypeDeclaration, tokens: Tokens): Seq[LocatableIdToken] = { if (tokens.isEmpty) return Seq.empty - val constant = tokenToId(tokens.head) + val constant = LocatableIdToken(tokens.head.contents, tokens.head.location) val field = FieldDeclaration(Array(), Array(Modifier(Tokens.StaticStr)), etd, constant) etd.appendField(field) Seq(constant) @@ -454,7 +431,7 @@ object Parse { throw new Exception(s"Unrecognised method ${tokens.toString()}") } - val id = tokenToId(tokens.get(startIndex)) + val id = LocatableIdToken(tokens.get(startIndex).contents, tokens.get(startIndex).location) val (index, formalParameterList) = parseFormalParameterList(startIndex + 1, tokens) if (index < tokens.length || formalParameterList.isEmpty) { throw new Exception(s"Unrecognised method ${tokens.toString()}") @@ -472,7 +449,7 @@ object Parse { ctd: IMutableTypeDeclaration ): Option[PropertyDeclaration] = { - val id = tokenToId(tokens.get(startIndex)) + val id = LocatableIdToken(tokens.get(startIndex).contents, tokens.get(startIndex).location) val index = startIndex + 1 if (index < tokens.length) { @@ -540,7 +517,7 @@ object Parse { var startLocation: Option[Location] = None var endLocation = Location.default while (index < tokens.length) { - val id = tokenToId(tokens.get(index)) + val id = LocatableIdToken(tokens.get(index).contents, tokens.get(index).location) val field = FieldDeclaration(md.annotations, md.modifiers, md.typeRef.get, id) ctd.appendField(field) @@ -557,11 +534,12 @@ object Parse { if (startLocation.isDefined) { // WARNING: I don't think there is a code path where this is not overwritten + // endByteOffset is now exclusive, so it already points one-past-the-end field.setBlockLocation( Position( startLocation.get.startLine, startLocation.get.startLineOffset, - startLocation.get.endByteOffset + 1 + startLocation.get.endByteOffset ), Position(endLocation.startLine, endLocation.startLineOffset, endLocation.startByteOffset) ) @@ -570,16 +548,11 @@ object Parse { fields.toSeq } - private def tokenToModifier(token: Token): Modifier = Modifier(token.contents) - - private def tokenToId(token: Token): LocatableIdToken = - LocatableIdToken(token.contents, token.location) - private def getId(startIndex: Int, tokens: Tokens): (Int, Option[LocatableIdToken]) = { if (startIndex >= tokens.length) { (startIndex, None) } else if (tokens.get(startIndex).isInstanceOf[LocatableIdToken]) { - val id = tokenToId(tokens.get(startIndex)) + val id = LocatableIdToken(tokens.get(startIndex).contents, tokens.get(startIndex).location) (startIndex + 1, Some(id)) } else { (startIndex, None) @@ -598,7 +571,7 @@ object Parse { ): (Int, Option[TypeNameSegment]) = { tokens(startIndex) match { case Some(token: LocatableIdToken) => - val id = tokenToId(token) + val id = LocatableIdToken(token.contents, token.location) val (nextIndex, typeArguments) = parseTypeArguments(startIndex + 1, tokens) (nextIndex, Some(TypeNameSegment(id, typeArguments))) case _ => (startIndex, None) @@ -673,6 +646,7 @@ object Parse { ): Int = { if (!tokens(startIndex).exists(_.matches(Tokens.AtSignStr))) return startIndex + val startLocation = tokens.get(startIndex).location val names = new mutable.ArrayBuffer[LocatableIdToken]() var (index, name) = getId(startIndex + 1, tokens) name.foreach(names.append) @@ -699,7 +673,11 @@ object Parse { Some(builder.toString()) } else None - accum.append(Annotation(qName.toString, parameters)) + // Capture full annotation location from @ through parameters + val endLocation = if (index > 0) tokens.get(index - 1).location else startLocation + val fullLocation = Location.span(startLocation, endLocation) + + accum.append(Annotation(qName.toString, parameters, Some(fullLocation))) index } @@ -738,7 +716,7 @@ object Parse { } else if (modifierTokenStrs.contains(tokens.get(index).contents.toLowerCase)) { if (modifiers == null) modifiers = new mutable.ArrayBuffer[Modifier]() - modifiers.append(tokenToModifier(tokens.get(index))) + modifiers.append(Modifier(tokens.get(index).contents, Some(tokens.get(index).location))) index += 1 } else if ( sharingModifiers.contains(tokens.get(index).contents.toLowerCase()) @@ -747,8 +725,13 @@ object Parse { // Combine to make sharing modifier if (modifiers == null) modifiers = new mutable.ArrayBuffer[Modifier]() + val spannedLocation = + Location.span(tokens.get(index).location, tokens.get(index + 1).location) modifiers.append( - Modifier(s"${tokens.get(index).contents} ${tokens.get(index + 1).contents}") + Modifier( + s"${tokens.get(index).contents} ${tokens.get(index + 1).contents}", + Some(spannedLocation) + ) ) index += 2 } else { diff --git a/shared/src/main/scala/com/financialforce/types/base/Annotation.scala b/shared/src/main/scala/com/financialforce/types/base/Annotation.scala index 11ce25b..19c26c2 100644 --- a/shared/src/main/scala/com/financialforce/types/base/Annotation.scala +++ b/shared/src/main/scala/com/financialforce/types/base/Annotation.scala @@ -3,20 +3,20 @@ */ package com.financialforce.types.base -import com.financialforce.types.ArrayInternCache - import scala.util.hashing.MurmurHash3 /** Annotation element, name is case-insensitive, parameters are unparsed. */ -case class Annotation(name: String, parameters: Option[String]) { +case class Annotation(name: String, parameters: Option[String], location: Option[Location] = None) { override def equals(obj: Any): Boolean = { val other = obj.asInstanceOf[Annotation] name.equalsIgnoreCase(other.name) && parameters.getOrElse("").equalsIgnoreCase(other.parameters.getOrElse("")) + // Note: location intentionally excluded from equality } override def hashCode(): Int = { MurmurHash3.orderedHash(Seq(name.toLowerCase(), parameters.getOrElse(""))) + // Note: location intentionally excluded from hash } override def toString: String = { @@ -24,13 +24,6 @@ case class Annotation(name: String, parameters: Option[String]) { } } -/** Caching support for Arrays of annotations. */ object Annotation { final val emptyArray = Array[Annotation]() - - private val cache = new ArrayInternCache[Annotation]() - - def intern(annotations: Array[Annotation]): Array[Annotation] = { - cache.intern(annotations) - } } diff --git a/shared/src/main/scala/com/financialforce/types/base/Modifier.scala b/shared/src/main/scala/com/financialforce/types/base/Modifier.scala index ef508ce..14f82ad 100644 --- a/shared/src/main/scala/com/financialforce/types/base/Modifier.scala +++ b/shared/src/main/scala/com/financialforce/types/base/Modifier.scala @@ -3,29 +3,22 @@ */ package com.financialforce.types.base -import com.financialforce.types.ArrayInternCache - /** Modifier element, text is case-insensitive. */ -case class Modifier(text: String) { +case class Modifier(text: String, location: Option[Location] = None) { override def equals(obj: Any): Boolean = { val other = obj.asInstanceOf[Modifier] text.equalsIgnoreCase(other.text) + // Note: location intentionally excluded from equality } override def hashCode(): Int = { text.toLowerCase.hashCode + // Note: location intentionally excluded from hash } override def toString: String = text } -/** Caching support for Arrays of modifiers. */ object Modifier { final val emptyArray = Array[Modifier]() - - private val cache = new ArrayInternCache[Modifier]() - - def intern(modifiers: Array[Modifier]): Array[Modifier] = { - cache.intern(modifiers) - } } diff --git a/shared/src/test/scala/com/financialforce/oparser/BufferTest.scala b/shared/src/test/scala/com/financialforce/oparser/BufferTest.scala new file mode 100644 index 0000000..56dc9f4 --- /dev/null +++ b/shared/src/test/scala/com/financialforce/oparser/BufferTest.scala @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2026 FinancialForce.com, inc. All rights reserved. + */ + +package com.financialforce.oparser + +import com.financialforce.types.base.Location +import org.scalatest.funspec.AnyFunSpec + +import java.nio.charset.StandardCharsets + +class BufferTest extends AnyFunSpec { + + describe("Buffer byte offset calculation") { + + it("should correctly calculate byte offsets for ASCII characters") { + val content = "ABC" + val buffer = new Buffer(content) + + // Append 3 ASCII characters (1 byte each) + // charOffset matches string positions, byteOffset tracks UTF-8 byte positions + buffer.append(byteOffset = 0, charOffset = 0, line = 0, lineOffset = 0, charByteLength = 1) + buffer.append(byteOffset = 1, charOffset = 1, line = 0, lineOffset = 1, charByteLength = 1) + buffer.append(byteOffset = 2, charOffset = 2, line = 0, lineOffset = 2, charByteLength = 1) + + val (captured, location) = buffer.captured() + + assert(captured == "ABC") + assert(location.startByteOffset == 0) + assert(location.endByteOffset == 3) // After last character (exclusive) + } + + it("should correctly calculate byte offsets for 2-byte UTF-8 characters") { + val content = "café" // é is 2 bytes in UTF-8 + val buffer = new Buffer(content) + + // c=1 byte, a=1 byte, f=1 byte, é=2 bytes + buffer.append(byteOffset = 0, charOffset = 0, line = 0, lineOffset = 0, charByteLength = 1) + buffer.append(byteOffset = 1, charOffset = 1, line = 0, lineOffset = 1, charByteLength = 1) + buffer.append(byteOffset = 2, charOffset = 2, line = 0, lineOffset = 2, charByteLength = 1) + buffer.append(byteOffset = 3, charOffset = 3, line = 0, lineOffset = 3, charByteLength = 2) // é is 2 bytes + + val (captured, location) = buffer.captured() + + assert(captured == "café") + assert(location.startByteOffset == 0) + assert(location.endByteOffset == 5) // 1+1+1+2 = 5 bytes total + } + + it("should correctly calculate byte offsets for 3-byte UTF-8 characters") { + val content = "中文" // Each Chinese character is 3 bytes in UTF-8 + val buffer = new Buffer(content) + + // 中=3 bytes, 文=3 bytes + buffer.append(byteOffset = 0, charOffset = 0, line = 0, lineOffset = 0, charByteLength = 3) + buffer.append(byteOffset = 3, charOffset = 1, line = 0, lineOffset = 1, charByteLength = 3) + + val (captured, location) = buffer.captured() + + assert(captured == "中文") + assert(location.startByteOffset == 0) + assert(location.endByteOffset == 6) // 3+3 = 6 bytes total + } + + it("should correctly calculate byte offsets for 4-byte UTF-8 characters (emoji)") { + val content = "A🤦B" // 🤦 is a surrogate pair, 4 bytes in UTF-8 + val buffer = new Buffer(content) + + // A=1 byte, 🤦=4 bytes (spans 2 char positions), B=1 byte + buffer.append(byteOffset = 0, charOffset = 0, line = 0, lineOffset = 0, charByteLength = 1) + buffer.append(byteOffset = 1, charOffset = 1, line = 0, lineOffset = 1, charByteLength = 4, charCount = 2) // emoji + buffer.append(byteOffset = 5, charOffset = 3, line = 0, lineOffset = 2, charByteLength = 1) // B at char position 3 + + val (captured, location) = buffer.captured() + + assert(captured == "A🤦B") + assert(location.startByteOffset == 0) + assert(location.endByteOffset == 6) // 1+4+1 = 6 bytes total + } + + it("should correctly calculate byte offsets for mixed ASCII and multi-byte characters") { + val content = "Hi😊" // H=1, i=1, 😊=4 bytes in UTF-8 (but 2 UTF-16 chars!) + val buffer = new Buffer(content) + + // In Java Strings (UTF-16), emoji is 2 chars: high surrogate (index 2) + low surrogate (index 3) + buffer.append(byteOffset = 0, charOffset = 0, line = 0, lineOffset = 0, charByteLength = 1) // 'H' + buffer.append(byteOffset = 1, charOffset = 1, line = 0, lineOffset = 1, charByteLength = 1) // 'i' + // Emoji: starts at charOffset 2, spans 2 char positions (surrogate pair), 4 bytes in UTF-8 + buffer.append(byteOffset = 2, charOffset = 2, line = 0, lineOffset = 2, charByteLength = 4, charCount = 2) + + val (captured, location) = buffer.captured() + + assert(captured == "Hi😊") // Now correctly captures the full emoji! + assert(location.startByteOffset == 0) + assert(location.endByteOffset == 6) // 1+1+4 = 6 bytes + } + + it("should handle clearing and reusing the buffer") { + val content = "Test" + val buffer = new Buffer(content) + + buffer.append(0, 0, 0, 0, 1) + buffer.append(1, 1, 0, 1, 1) + + val (captured1, location1) = buffer.captured() + assert(captured1 == "Te") + assert(location1.endByteOffset == 2) + + buffer.clear() + buffer.append(2, 2, 0, 2, 1) + buffer.append(3, 3, 0, 3, 1) + + val (captured2, location2) = buffer.captured() + assert(captured2 == "st") + assert(location2.startByteOffset == 2) + assert(location2.endByteOffset == 4) + } + + it("should verify byte offsets match actual UTF-8 encoding") { + // This test verifies our byte offset calculation matches actual UTF-8 encoding + val testCases = Seq( + ("A", 1), // ASCII + ("é", 2), // 2-byte UTF-8 + ("中", 3), // 3-byte UTF-8 + ("🤦", 4) // 4-byte UTF-8 (surrogate pair) + ) + + testCases.foreach { case (char, expectedBytes) => + val actualBytes = char.getBytes(StandardCharsets.UTF_8).length + assert( + actualBytes == expectedBytes, + s"Character '$char' should be $expectedBytes bytes but was $actualBytes" + ) + } + } + } +} \ No newline at end of file diff --git a/shared/src/test/scala/com/financialforce/oparser/Compare.scala b/shared/src/test/scala/com/financialforce/oparser/Compare.scala index a407ad0..0bbcb52 100644 --- a/shared/src/test/scala/com/financialforce/oparser/Compare.scala +++ b/shared/src/test/scala/com/financialforce/oparser/Compare.scala @@ -4,8 +4,23 @@ package com.financialforce.oparser +import com.financialforce.types.base.Location + object Compare { + // Compare locations ignoring byte offsets (ANTLR doesn't calculate them for performance) + private def locationsMatch(loc1: Option[Location], loc2: Option[Location]): Boolean = { + (loc1, loc2) match { + case (Some(l1), Some(l2)) => + l1.startLine == l2.startLine && + l1.startLineOffset == l2.startLineOffset && + l1.endLine == l2.endLine && + l1.endLineOffset == l2.endLineOffset + case (None, None) => true + case _ => false + } + } + def compareClassTypeDeclarations( first: TestClassTypeDeclaration, second: TestClassTypeDeclaration @@ -78,11 +93,29 @@ object Compare { .mkString("Array(", ", ", ")")} != ${second._annotations.mkString("Array(", ", ", ")")}") } + // Compare annotation locations (ignoring byte offsets) + first._annotations.zip(second._annotations).foreach { case (a1, a2) => + if (!locationsMatch(a1.location, a2.location)) { + throw new Exception( + s"Different annotation location for '${a1.name}': ${a1.location} != ${a2.location}" + ) + } + } + if (!(first._modifiers sameElements second._modifiers)) { throw new Exception(s"Different modifiers ${first._modifiers .mkString("Array(", ", ", ")")} != ${second._modifiers.mkString("Array(", ", ", ")")}") } + // Compare modifier locations (ignoring byte offsets) + first._modifiers.zip(second._modifiers).foreach { case (m1, m2) => + if (!locationsMatch(m1.location, m2.location)) { + throw new Exception( + s"Different modifier location for '${m1.text}': ${m1.location} != ${m2.location}" + ) + } + } + if (first.id != second.id) { throw new Exception(s"Different or empty class id ${first.id} != ${second.id}") } @@ -149,11 +182,29 @@ object Compare { .mkString("Array(", ", ", ")")} != ${second.annotations.mkString("Array(", ", ", ")")}") } + // Compare annotation locations (ignoring byte offsets) + first.annotations.zip(second.annotations).foreach { case (a1, a2) => + if (!locationsMatch(a1.location, a2.location)) { + throw new Exception( + s"Different annotation location for '${a1.name}': ${a1.location} != ${a2.location}" + ) + } + } + if (!(first.modifiers sameElements second.modifiers)) { throw new Exception(s"Different modifiers ${first.modifiers .mkString("Array(", ", ", ")")} != ${second.modifiers.mkString("Array(", ", ", ")")}") } + // Compare modifier locations (ignoring byte offsets) + first.modifiers.zip(second.modifiers).foreach { case (m1, m2) => + if (!locationsMatch(m1.location, m2.location)) { + throw new Exception( + s"Different modifier location for '${m1.text}': ${m1.location} != ${m2.location}" + ) + } + } + if (first.id != second.id) { throw new Exception(s"Different or empty interface id ${first.id} != ${second.id}") } @@ -179,11 +230,29 @@ object Compare { .mkString("Array(", ", ", ")")} != ${second.annotations.mkString("Array(", ", ", ")")}") } + // Compare annotation locations (ignoring byte offsets) + first.annotations.zip(second.annotations).foreach { case (a1, a2) => + if (!locationsMatch(a1.location, a2.location)) { + throw new Exception( + s"Different annotation location for '${a1.name}': ${a1.location} != ${a2.location}" + ) + } + } + if (!(first.modifiers sameElements second.modifiers)) { throw new Exception(s"Different modifiers ${first.modifiers .mkString("Array(", ", ", ")")} != ${second.modifiers.mkString("Array(", ", ", ")")}") } + // Compare modifier locations (ignoring byte offsets) + first.modifiers.zip(second.modifiers).foreach { case (m1, m2) => + if (!locationsMatch(m1.location, m2.location)) { + throw new Exception( + s"Different modifier location for '${m1.text}': ${m1.location} != ${m2.location}" + ) + } + } + if (first.id != second.id) { throw new Exception(s"Different or empty enum id ${first.id} != ${second.id}") } diff --git a/shared/src/test/scala/com/financialforce/oparser/SmokeTest.scala b/shared/src/test/scala/com/financialforce/oparser/SmokeTest.scala index f4af536..d4bdd05 100644 --- a/shared/src/test/scala/com/financialforce/oparser/SmokeTest.scala +++ b/shared/src/test/scala/com/financialforce/oparser/SmokeTest.scala @@ -19,8 +19,9 @@ class SmokeTest extends AnyFunSpec { def extractLocation(contents: String, location: Location): String = { val bytes = contents.getBytes(StandardCharsets.UTF_8) + // Location uses [start, end) convention with exclusive endByteOffset new String( - bytes.slice(location.startByteOffset, location.endByteOffset + 1), + bytes.slice(location.startByteOffset, location.endByteOffset), StandardCharsets.UTF_8 ) } @@ -162,7 +163,7 @@ class SmokeTest extends AnyFunSpec { val thrown = intercept[Exception] { parse(content) } - assert(thrown.getMessage == "Unrecognised method [2.3 -> 2.16] public Dummy ( )") + assert(thrown.getMessage == "Unrecognised method [2.2 -> 2.16] public Dummy ( )") } it("errors on constructor without body terminated with comment & semi-colon") { @@ -177,7 +178,7 @@ class SmokeTest extends AnyFunSpec { val thrown = intercept[Exception] { parse(content) } - assert(thrown.getMessage == "Unrecognised method [2.3 -> 2.16] public Dummy ( )") + assert(thrown.getMessage == "Unrecognised method [2.2 -> 2.16] public Dummy ( )") } it("errors on constructor without body terminated by class end") { diff --git a/shared/src/test/scala/com/financialforce/oparser/testutil/AntlrParser.scala b/shared/src/test/scala/com/financialforce/oparser/testutil/AntlrParser.scala index 89d8e0b..00744e6 100644 --- a/shared/src/test/scala/com/financialforce/oparser/testutil/AntlrParser.scala +++ b/shared/src/test/scala/com/financialforce/oparser/testutil/AntlrParser.scala @@ -4,13 +4,13 @@ package com.financialforce.oparser.testutil import com.financialforce.oparser._ +import com.financialforce.oparser.testutil.AntlrOps._ import com.financialforce.types.base._ import com.financialforce.types.{ITypeDeclaration, base} -import com.nawforce.apexparser.ApexParser import com.nawforce.runtime.parsers.{CodeParser, SourceData} import com.nawforce.runtime.platform.Path -import com.financialforce.oparser.testutil.AntlrOps._ -import com.nawforce.apexparser.ApexParser.ModifierContext +import io.github.apexdevtools.apexparser.ApexParser +import io.github.apexdevtools.apexparser.ApexParser.ModifierContext import scala.collection.immutable.ArraySeq import scala.collection.mutable @@ -27,7 +27,8 @@ object AntlrParser { def parse(path: String, contents: Array[Byte]): Option[ITypeDeclaration] = { // We re-use apex-ls CodeParser for JS/JVM portability - val codeParser = CodeParser(Path(path), SourceData(new String(contents))) + // Pass bytes directly to SourceData to ensure byte offsets match the original file + val codeParser = CodeParser(Path(path), SourceData(contents)) val issuesAndCU = codeParser.parseClass() issuesAndCU.issues.headOption.map(issue => throw new Exception(issue.asString)) @@ -72,9 +73,12 @@ object AntlrParser { val len = modifier.length // Add a space back in, it may not have been a single space but very likely it was if (len >= 7 && modifier.toLowerCase.endsWith("sharing")) { - Modifier(modifier.substring(0, len - 7) + " " + modifier.substring(len - 7)) + Modifier( + modifier.substring(0, len - 7) + " " + modifier.substring(len - 7), + Some(ctx.location) + ) } else - Modifier(modifier) + Modifier(modifier, Some(ctx.location)) } def antlrAnnotation(ctx: ApexParser.AnnotationContext): Annotation = { @@ -86,7 +90,7 @@ object AntlrParser { .map(CodeParser.getText) .orElse(CodeParser.toScala(ctx.elementValuePairs()).map(CodeParser.getText)) .orElse(if (CodeParser.getText(ctx).endsWith("()")) Some("") else None) - Annotation(qName.toString, args) + Annotation(qName.toString, args, Some(ctx.location)) } def antlrTypeList(ctx: ApexParser.TypeListContext): ArraySeq[TypeRef] = {