some more infrastructure for macro profiling

Author: ghik

build.sbt

@@ -364,16 +364,17 @@ lazy val `commons-comprof` = project
       s"-P:scalac-profiling:sourceroot:${baseDirectory.value}",
       "-P:scalac-profiling:generate-macro-flamegraph",
       "-P:scalac-profiling:no-profiledb",
-      "-Ystatistics",
+      "-Xmacro-settings:statsEnabled",
+      "-Ystatistics:typer",
     ),
     sourceGenerators in Compile += Def.task {
-      val originalSrc = (sourceDirectory in `commons-core`).value / 
-        "main/scala/com/avsystem/commons/rest/openapi/OpenApi.scala"
+      val originalSrc = (sourceDirectory in `commons-core`).value /
+        "test/scala/com/avsystem/commons/rest/RestTestApi.scala"
       val originalContent = IO.read(originalSrc)
-      (0 until 10).map { i =>
+      (0 until 100).map { i =>
         val pkg = f"oa$i%02d"
-        val newContent = originalContent.replaceAllLiterally("package rest.openapi", s"package rest.$pkg")
-        val newFile = (sourceManaged in Compile).value / pkg / "OpenApi.scala"
+        val newContent = originalContent.replaceAllLiterally("package rest", s"package rest\npackage $pkg")
+        val newFile = (sourceManaged in Compile).value / pkg / "RestTestApi.scala"
         IO.write(newFile, newContent)
         newFile
       }

commons-core/src/test/scala/com/avsystem/commons/rest/RestTestApi.scala

@@ -17,7 +17,7 @@ object FlatBaseEntity extends RestDataCompanion[FlatBaseEntity]
 @description("REST entity")
 case class RestEntity(
   @description("entity id") id: String,
-  @whenAbsent("anonymous") name: String = whenAbsent.debugMacro.value,
+  @whenAbsent("anonymous") name: String = whenAbsent.value,
   @description("recursive optional subentity") subentity: OptArg[RestEntity] = OptArg.Empty
 ) extends FlatBaseEntity
 object RestEntity extends RestDataCompanion[RestEntity]

commons-macros/src/main/scala/com/avsystem/commons/macros/MacroCommons.scala

@@ -48,14 +48,14 @@ trait MacroCommons { bundle =>
   final lazy val MapSym = typeOf[scala.collection.immutable.Map[_, _]].typeSymbol
   final lazy val FutureSym = typeOf[scala.concurrent.Future[_]].typeSymbol
   final lazy val OptionClass = definitions.OptionClass
-  final lazy val AnnotationAggregateType = getType(tq"$CommonsPkg.annotation.AnnotationAggregate")
-  final lazy val DefaultsToNameAT = getType(tq"$CommonsPkg.annotation.defaultsToName")
-  final lazy val InferAT: Type = getType(tq"$CommonsPkg.meta.infer")
-  final lazy val NotInheritedFromSealedTypes = getType(tq"$CommonsPkg.annotation.NotInheritedFromSealedTypes")
+  final lazy val AnnotationAggregateType = staticType(tq"$CommonsPkg.annotation.AnnotationAggregate")
+  final lazy val DefaultsToNameAT = staticType(tq"$CommonsPkg.annotation.defaultsToName")
+  final lazy val InferAT: Type = staticType(tq"$CommonsPkg.meta.infer")
+  final lazy val NotInheritedFromSealedTypes = staticType(tq"$CommonsPkg.annotation.NotInheritedFromSealedTypes")
   final lazy val SeqCompanionSym = typeOf[scala.collection.Seq.type].termSymbol
-  final lazy val PositionedAT = getType(tq"$CommonsPkg.annotation.positioned")
-  final lazy val ImplicitNotFoundAT = getType(tq"$ScalaPkg.annotation.implicitNotFound")
-  final lazy val ImplicitNotFoundSym = getType(tq"$MiscPkg.ImplicitNotFound[_]").typeSymbol
+  final lazy val PositionedAT = staticType(tq"$CommonsPkg.annotation.positioned")
+  final lazy val ImplicitNotFoundAT = staticType(tq"$ScalaPkg.annotation.implicitNotFound")
+  final lazy val ImplicitNotFoundSym = staticType(tq"$MiscPkg.ImplicitNotFound[_]").typeSymbol
 
   final lazy val NothingTpe: Type = typeOf[Nothing]
   final lazy val StringPFTpe: Type = typeOf[PartialFunction[String, Any]]
@@ -72,12 +72,12 @@ trait MacroCommons { bundle =>
     definitions.ScalaPackageClass.toType.member(TermName("scalajs")) != NoSymbol
 
   final lazy val ownerChain = {
-    val sym = c.typecheck(q"val ${c.freshName(TermName(""))} = null").symbol
+    val sym = typecheck(q"val ${c.freshName(TermName(""))} = null").symbol
     Iterator.iterate(sym)(_.owner).takeWhile(_ != NoSymbol).drop(1).toList
   }
 
   final lazy val enclosingClasses = {
-    val enclosingSym = c.typecheck(q"this", silent = true) match {
+    val enclosingSym = typecheck(q"this", silent = true) match {
       case EmptyTree => Iterator.iterate(c.internal.enclosingOwner)(_.owner).find(_.isModuleClass).get
       case tree => tree.tpe.typeSymbol
     }
@@ -89,16 +89,51 @@ trait MacroCommons { bundle =>
     case _ => false
   }
 
+  final val statsEnabled: Boolean =
+    c.settings.contains("statsEnabled")
+
   def debug(msg: => String): Unit =
     if (debugEnabled) {
       error(msg)
     }
 
-  def showOnDebug(tree: Tree): Tree = {
-    debug(show(tree))
-    tree
+  def instrument(tree: => Tree): Tree = measure(c.macroApplication.symbol.fullName) {
+    val result = tree
+    debug(show(result))
+    result
   }
 
+  private[this] var measureStack = List.empty[String]
+
+  def measure[T](what: String)(expr: => T): T =
+    if (!statsEnabled || measureStack.contains(what)) expr else {
+      measureStack ::= what
+      val start = System.nanoTime()
+      try expr finally {
+        echo(s"$what ${(System.nanoTime() - start) / 1000}")
+        measureStack = measureStack.tail
+      }
+    }
+
+  def inferImplicitValue(
+    pt: Type,
+    silent: Boolean = true,
+    withMacrosDisabled: Boolean = false,
+    pos: Position = c.enclosingPosition
+  ): Tree = {
+    debug(s"macro implicit search for $pt")
+    measure("implicit")(c.inferImplicitValue(pt, silent, withMacrosDisabled, pos))
+  }
+
+  def typecheck(
+    tree: Tree,
+    mode: c.TypecheckMode = c.TERMmode,
+    pt: Type = WildcardType,
+    silent: Boolean = false,
+    withImplicitViewsDisabled: Boolean = false,
+    withMacrosDisabled: Boolean = false
+  ): Tree = measure("typecheck")(c.typecheck(tree, mode, pt, silent, withImplicitViewsDisabled, withMacrosDisabled))
+
   def containsInaccessibleThises(tree: Tree): Boolean = tree.exists {
     case t@This(_) if !t.symbol.isPackageClass && !enclosingClasses.contains(t.symbol) => true
     case _ => false
@@ -232,7 +267,8 @@ trait MacroCommons { bundle =>
     if (withSupers) withSuperSymbols(s) else Iterator(s)
 
   def allAnnotations(s: Symbol, tpeFilter: Type,
-    seenFrom: Type = NoType, withInherited: Boolean = true, fallback: List[Tree] = Nil): List[Annot] = {
+    seenFrom: Type = NoType, withInherited: Boolean = true, fallback: List[Tree] = Nil
+  ): List[Annot] = measure("annotations") {
 
     val initSym = orConstructorParam(s)
     def inherited(annot: Annotation, superSym: Symbol): Boolean =
@@ -247,8 +283,8 @@ trait MacroCommons { bundle =>
   }
 
   def findAnnotation(s: Symbol, tpe: Type,
-    seenFrom: Type = NoType, withInherited: Boolean = true, fallback: List[Tree] = Nil): Option[Annot] = {
-
+    seenFrom: Type = NoType, withInherited: Boolean = true, fallback: List[Tree] = Nil
+  ): Option[Annot] = measure("annotations") {
     val initSym = orConstructorParam(s)
     def find(annots: List[Annot]): Option[Annot] = annots match {
       case head :: tail =>
@@ -283,7 +319,7 @@ trait MacroCommons { bundle =>
     ownerChain.filter(_.isConstructor).map(_.owner.asClass.module).find(_ != NoSymbol).getOrElse(NoSymbol)
 
   lazy val companionReplacement: Symbol =
-    c.typecheck(q"$companionReplacementName", silent = true) match {
+    typecheck(q"$companionReplacementName", silent = true) match {
       case EmptyTree => NoSymbol
       case _ => enclosingConstructorCompanion
     }
@@ -371,7 +407,7 @@ trait MacroCommons { bundle =>
 
   def tryInferCachedImplicit(tpe: Type): Option[TermName] = {
     def compute: Option[TermName] =
-      Option(c.inferImplicitValue(tpe)).filter(_ != EmptyTree).map { found =>
+      Option(inferImplicitValue(tpe)).filter(_ != EmptyTree).map { found =>
         def newCachedImplicit(): TermName = {
           val name = c.freshName(TermName("cachedImplicit"))
           inferredImplicitTypes(name) = found.tpe
@@ -456,7 +492,7 @@ trait MacroCommons { bundle =>
     }
 
   def implicitNotFoundMsg(tpe: Type): String =
-    implicitNotFoundMsg(Nil, tpe, c.inferImplicitValue(getType(tq"$ImplicitNotFoundCls[$tpe]")))
+    implicitNotFoundMsg(Nil, tpe, inferImplicitValue(getType(tq"$ImplicitNotFoundCls[$tpe]")))
 
   class treeOps[T <: Tree](t: T) {
     def debug: T = {
@@ -512,7 +548,7 @@ trait MacroCommons { bundle =>
   def warning(msg: String): Unit =
     c.warning(c.enclosingPosition, msg)
 
-  def ensure(condition: Boolean, msg: String): Unit =
+  def ensure(condition: Boolean, msg: => String): Unit =
     if (!condition) {
       abort(msg)
     }
@@ -583,8 +619,11 @@ trait MacroCommons { bundle =>
     }
   }
 
+  def staticType(tpeTree: Tree): Type =
+    getType(tpeTree)
+
   def getType(typeTree: Tree): Type =
-    c.typecheck(typeTree, c.TYPEmode).tpe
+    measure("typecheck.getType")(typecheck(typeTree, c.TYPEmode).tpe)
 
   def pathTo(sym: Symbol): Tree =
     if (enclosingClasses.contains(sym)) This(sym)
@@ -937,7 +976,7 @@ trait MacroCommons { bundle =>
   def applyUnapplyFor(tpe: Type): Option[ApplyUnapply] =
     typedCompanionOf(tpe).flatMap(comp => applyUnapplyFor(tpe, comp))
 
-  def applyUnapplyFor(tpe: Type, typedCompanion: Tree): Option[ApplyUnapply] = {
+  def applyUnapplyFor(tpe: Type, typedCompanion: Tree): Option[ApplyUnapply] = measure("applyUnapplyFor") {
     val dtpe = tpe.dealias
     val ts = dtpe.typeSymbol.asType
     val caseClass = ts.isClass && ts.asClass.isCaseClass
@@ -990,7 +1029,7 @@ trait MacroCommons { bundle =>
     }
   }
 
-  def singleValueFor(tpe: Type): Option[Tree] = tpe match {
+  def singleValueFor(tpe: Type): Option[Tree] = measure("singleValueFor")(tpe match {
     case ThisType(sym) if enclosingClasses.contains(sym) =>
       Some(This(sym))
     case ThisType(sym) if sym.isModuleClass =>
@@ -1007,15 +1046,15 @@ trait MacroCommons { bundle =>
       singleValueFor(pre).map(prefix => Select(prefix, sym.asClass.module))
     case _ =>
       None
-  }
+  })
 
   def typedCompanionOf(tpe: Type): Option[Tree] = {
     val result = tpe match {
       case TypeRef(pre, sym, _) if sym.companion != NoSymbol =>
         singleValueFor(pre).map(Select(_, sym.companion)) orElse singleValueFor(tpe.companion)
       case _ => singleValueFor(tpe.companion)
     }
-    result.map(c.typecheck(_))
+    result.map(typecheck(_))
   }
 
   def typeOfTypeSymbol(sym: TypeSymbol): Type = sym.toType match {
@@ -1056,7 +1095,7 @@ trait MacroCommons { bundle =>
           s"it resides in separate file than macro invocation and has no @positioned annotation")
       })
 
-  def knownSubtypes(tpe: Type, ordered: Boolean = false): Option[List[Type]] = {
+  def knownSubtypes(tpe: Type, ordered: Boolean = false): Option[List[Type]] = measure("knownSubtypes") {
     val dtpe = tpe.dealias
     val (tpeSym, refined) = dtpe match {
       case RefinedType(List(single), scope) =>
@@ -1093,7 +1132,7 @@ trait MacroCommons { bundle =>
     val methodName = c.freshName(TermName("m"))
     val typeDefs = typeParams.map(typeSymbolToTypeDef(_, forMethod = true))
 
-    val tree = c.typecheck(
+    val tree = typecheck(
       q"""
         def $methodName[..$typeDefs](f: ${treeForType(undetTpe)} => $UnitCls): $UnitCls = ()
         $methodName((_: $detTpe) => ())

commons-macros/src/main/scala/com/avsystem/commons/macros/TestMacros.scala

@@ -56,7 +56,7 @@ class TestMacros(val c: blackbox.Context) extends TypeClassDerivation {
     q"$PredefObj.???"
   }
 
-  def testKnownSubtypes[T: WeakTypeTag, R: WeakTypeTag]: Tree = showOnDebug {
+  def testKnownSubtypes[T: WeakTypeTag, R: WeakTypeTag]: Tree = instrument {
     val expectedResultTpe = knownSubtypes(weakTypeOf[T])
       .map(types => getType(tq"(..$types)"))
       .getOrElse(typeOf[Nothing])
@@ -67,7 +67,7 @@ class TestMacros(val c: blackbox.Context) extends TypeClassDerivation {
 
   val ApplierUnapplierCls = tq"$CommonsPkg.macros.ApplierUnapplier"
 
-  def applierUnapplier[T: WeakTypeTag, F: WeakTypeTag]: Tree = showOnDebug {
+  def applierUnapplier[T: WeakTypeTag, F: WeakTypeTag]: Tree = instrument {
     val ttpe = weakTypeOf[T]
     val ftpe = weakTypeOf[F]
 

commons-macros/src/main/scala/com/avsystem/commons/macros/TypeClassDerivation.scala

@@ -197,15 +197,15 @@ trait TypeClassDerivation extends MacroCommons {
     }
   }
 
-  def materialize[T: WeakTypeTag]: Tree = showOnDebug {
+  def materialize[T: WeakTypeTag]: Tree = instrument {
     val tpe = weakTypeOf[T]
     withRecursiveImplicitGuard(tpe, materializeFor(tpe))
   }
 
   def materializeImplicitly[T: WeakTypeTag](allow: Tree): Tree =
-    showOnDebug(materialize[T])
+    instrument(materialize[T])
 
-  def materializeMacroGenerated[T: WeakTypeTag]: Tree = showOnDebug {
+  def materializeMacroGenerated[T: WeakTypeTag]: Tree = instrument {
     val tpe = weakTypeOf[T].dealias
     val companionTpe = c.macroApplication.tpe.dealias.typeArgs.head
     val tcTpe = typeClassInstance(tpe)

commons-macros/src/main/scala/com/avsystem/commons/macros/meta/AdtMetadataMacros.scala

@@ -11,9 +11,9 @@ class AdtMetadataMacros(ctx: blackbox.Context) extends AbstractMacroCommons(ctx)
 
   import c.universe._
 
-  final lazy val AdtParamMetadataAT: Type = getType(tq"$MetaPackage.adtParamMetadata")
-  final lazy val AdtCaseMetadataAT: Type = getType(tq"$MetaPackage.adtCaseMetadata")
-  final lazy val ReifyDefaultValueAT: Type = getType(tq"$MetaPackage.reifyDefaultValue")
+  final lazy val AdtParamMetadataAT: Type = staticType(tq"$MetaPackage.adtParamMetadata")
+  final lazy val AdtCaseMetadataAT: Type = staticType(tq"$MetaPackage.adtCaseMetadata")
+  final lazy val ReifyDefaultValueAT: Type = staticType(tq"$MetaPackage.reifyDefaultValue")
 
   sealed trait AdtSymbol extends MacroSymbol with SelfMatchedSymbol {
     def tpe: Type
@@ -287,7 +287,7 @@ class AdtMetadataMacros(ctx: blackbox.Context) extends AbstractMacroCommons(ctx)
     }
   }
 
-  def materialize[Real: WeakTypeTag]: Tree = showOnDebug {
+  def materialize[Real: WeakTypeTag]: Tree = instrument {
     val adtTpe = weakTypeOf[Real].dealias
     val metadataTpe = c.macroApplication.tpe.dealias
     materializeMetadata(adtTpe, metadataTpe)
@@ -308,7 +308,7 @@ class AdtMetadataMacros(ctx: blackbox.Context) extends AbstractMacroCommons(ctx)
     }
   }
 
-  def materializeMacroGenerated[Real: WeakTypeTag]: Tree = showOnDebug {
+  def materializeMacroGenerated[Real: WeakTypeTag]: Tree = instrument {
     val adtTpe = weakTypeOf[Real].dealias
     val List(companionTpe, metadataTpe) = c.macroApplication.tpe.dealias.typeArgs
     mkMacroGenerated(companionTpe, metadataTpe, q"${c.prefix}.materialize[$adtTpe]")

commons-macros/src/main/scala/com/avsystem/commons/macros/meta/MacroMetadatas.scala

@@ -10,17 +10,17 @@ trait MacroMetadatas extends MacroSymbols {
   import c.universe._
 
   val ParamPositionObj: Tree = q"$MetaPackage.ParamPosition"
-  val TypedMetadataType: Type = getType(tq"$MetaPackage.TypedMetadata[_]")
-  val MetadataParamStrategyType: Type = getType(tq"$MetaPackage.MetadataParamStrategy")
-  val ReifyAnnotAT: Type = getType(tq"$MetaPackage.reifyAnnot")
-  val IsAnnotatedAT: Type = getType(tq"$MetaPackage.isAnnotated[_]")
-  val ReifyNameAT: Type = getType(tq"$MetaPackage.reifyName")
-  val ReifyPositionAT: Type = getType(tq"$MetaPackage.reifyPosition")
-  val ReifyFlagsAT: Type = getType(tq"$MetaPackage.reifyFlags")
-  val CheckedAT: Type = getType(tq"$MetaPackage.checked")
-  val ParamPositionTpe: Type = getType(tq"$MetaPackage.ParamPosition")
-  val ParamFlagsTpe: Type = getType(tq"$MetaPackage.ParamFlags")
-  val TypeFlagsTpe: Type = getType(tq"$MetaPackage.TypeFlags")
+  val TypedMetadataType: Type = staticType(tq"$MetaPackage.TypedMetadata[_]")
+  val MetadataParamStrategyType: Type = staticType(tq"$MetaPackage.MetadataParamStrategy")
+  val ReifyAnnotAT: Type = staticType(tq"$MetaPackage.reifyAnnot")
+  val IsAnnotatedAT: Type = staticType(tq"$MetaPackage.isAnnotated[_]")
+  val ReifyNameAT: Type = staticType(tq"$MetaPackage.reifyName")
+  val ReifyPositionAT: Type = staticType(tq"$MetaPackage.reifyPosition")
+  val ReifyFlagsAT: Type = staticType(tq"$MetaPackage.reifyFlags")
+  val CheckedAT: Type = staticType(tq"$MetaPackage.checked")
+  val ParamPositionTpe: Type = staticType(tq"$MetaPackage.ParamPosition")
+  val ParamFlagsTpe: Type = staticType(tq"$MetaPackage.ParamFlags")
+  val TypeFlagsTpe: Type = staticType(tq"$MetaPackage.TypeFlags")
 
   def actualMetadataType(baseMetadataType: Type, realType: Type, realTypeDesc: String, verbatim: Boolean): Res[Type] = {
     val (wildcards, underlying) = baseMetadataType match {

commons-macros/src/main/scala/com/avsystem/commons/macros/meta/MacroSymbols.scala

@@ -15,14 +15,14 @@ trait MacroSymbols extends MacroCommons {
   val RpcUtils = q"$RpcPackage.RpcUtils"
   val OptionLikeCls = tq"$MetaPackage.OptionLike"
   val CanBuildFromCls = tq"$CollectionPkg.generic.CanBuildFrom"
-  val RpcArityAT: Type = getType(tq"$MetaPackage.SymbolArity")
-  val SingleArityAT: Type = getType(tq"$MetaPackage.single")
-  val OptionalArityAT: Type = getType(tq"$MetaPackage.optional")
-  val MultiArityAT: Type = getType(tq"$MetaPackage.multi")
-  val CompositeAT: Type = getType(tq"$MetaPackage.composite")
-  val AuxiliaryAT: Type = getType(tq"$MetaPackage.auxiliary")
-  val AnnotatedAT: Type = getType(tq"$MetaPackage.annotated[_]")
-  val TaggedAT: Type = getType(tq"$RpcPackage.tagged[_]")
+  val RpcArityAT: Type = staticType(tq"$MetaPackage.SymbolArity")
+  val SingleArityAT: Type = staticType(tq"$MetaPackage.single")
+  val OptionalArityAT: Type = staticType(tq"$MetaPackage.optional")
+  val MultiArityAT: Type = staticType(tq"$MetaPackage.multi")
+  val CompositeAT: Type = staticType(tq"$MetaPackage.composite")
+  val AuxiliaryAT: Type = staticType(tq"$MetaPackage.auxiliary")
+  val AnnotatedAT: Type = staticType(tq"$MetaPackage.annotated[_]")
+  val TaggedAT: Type = staticType(tq"$RpcPackage.tagged[_]")
   val WhenUntaggedArg: Symbol = TaggedAT.member(TermName("whenUntagged"))
 
   def primaryConstructor(ownerType: Type, ownerParam: Option[MacroSymbol]): Symbol =

commons-macros/src/main/scala/com/avsystem/commons/macros/misc/BidirectionalMacro.scala

@@ -121,7 +121,7 @@ class BidirectionalMacro(ctx: blackbox.Context) extends AbstractMacroCommons(ctx
         CaseDef(c.untypecheck(exprToPattern(body, boundNamesSet)), guard, c.untypecheck(patternToExpr(pattern)))
     }
 
-    val reversed = c.typecheck(Match(EmptyTree, cases.init.map(reverseCaseDef)), pt = weakTypeOf[PartialFunction[B, A]])
+    val reversed = typecheck(Match(EmptyTree, cases.init.map(reverseCaseDef)), pt = weakTypeOf[PartialFunction[B, A]])
     q"($pf, $reversed)"
   }
 

commons-macros/src/main/scala/com/avsystem/commons/macros/misc/DelegationMacros.scala

@@ -11,7 +11,7 @@ class DelegationMacros(ctx: blackbox.Context) extends AbstractMacroCommons(ctx)
 
   final def DelegationCls: Tree = tq"$MiscPkg.Delegation"
 
-  def delegate[A: WeakTypeTag, B: WeakTypeTag](source: Tree): Tree = showOnDebug {
+  def delegate[A: WeakTypeTag, B: WeakTypeTag](source: Tree): Tree = instrument {
     val targetTpe = weakTypeOf[B]
 
     val targetSymbol = targetTpe.dealias.typeSymbol
@@ -54,7 +54,7 @@ class DelegationMacros(ctx: blackbox.Context) extends AbstractMacroCommons(ctx)
      """
   }
 
-  def materializeDelegation[A: WeakTypeTag, B: WeakTypeTag]: Tree = showOnDebug {
+  def materializeDelegation[A: WeakTypeTag, B: WeakTypeTag]: Tree = instrument {
     val targetTpe = weakTypeOf[B]
     val sourceTpe = weakTypeOf[A]