Merge pull request #542 from dwijnand/bidirection

Tweak setup for forward compatibility testing

Author: dwijnand

functional-tests/src/main/scala/com/typesafe/tools/mima/lib/AppRunTest.scala

@@ -1,20 +1,26 @@
 package com.typesafe.tools.mima.lib
 
+import scala.reflect.io.{ Directory, Path }
 import scala.util.{ Failure, Success, Try }
 
-/** Test running the App, using library v2. */
+/** Test running the App, using library v1 or v2. */
 object AppRunTest {
-  def main(args: Array[String]): Unit = TestCli.argsToTests(args.toList, testAppRun).unsafeRunTest()
+  def testAppRun(testCase: TestCase, direction: Direction): Try[Unit] = {
+    testAppRun1(testCase, direction.lhs(testCase), direction.rhs(testCase), direction.oracleFile)
+  }
 
-  def testAppRun(testCase: TestCase): Try[Unit] = for {
-    () <- testCase.compileThem
-    pending = testCase.versionedFile("testAppRun.pending").exists
-    emptyPT = testCase.blankFile(testCase.versionedFile("problems.txt"))
-    () <- testCase.runMain(testCase.outV1) // expect at least v1 to pass, even in the "pending" case
-    () <- testCase.runMain(testCase.outV2) match {
-      case Failure(t)  if !pending &&  emptyPT => Failure(t)
-      case Success(()) if !pending && !emptyPT => Failure(new Exception("expected running App to fail"))
-      case _                                   => Success(())
+  def testAppRun1(testCase: TestCase, v1: Directory, v2: Directory, oracleFile: Path): Try[Unit] = for {
+    () <- testCase.compileBoth
+    pending  = testCase.versionedFile("testAppRun.pending").exists
+    expectOk = testCase.blankFile(testCase.versionedFile(oracleFile))
+  //() <- testCase.compileApp(v2)      // compile app with v2
+  //() <- testCase.runMain(v2)         // sanity check 1: run app with v2
+    () <- testCase.compileApp(v1)      // recompile app with v1
+    () <- testCase.runMain(v1)         // sanity check 2: run app with v1
+    () <- testCase.runMain(v2) match { // test: run app, compiled with v1, with v2
+      case Failure(t)  if !pending &&  expectOk => Failure(t)
+      case Success(()) if !pending && !expectOk => Failure(new Exception("expected running App to fail"))
+      case _                                    => Success(())
     }
   } yield ()
 }

functional-tests/src/main/scala/com/typesafe/tools/mima/lib/CollectProblemsTest.scala

@@ -9,12 +9,10 @@ import scala.collection.JavaConverters._
 import scala.util.{ Failure, Success, Try }
 
 object CollectProblemsTest {
-  def main(args: Array[String]): Unit = TestCli.argsToTests(args.toList, testCollectProblems).unsafeRunTest()
-
-  def testCollectProblems(testCase: TestCase): Try[Unit] = for {
-    () <- testCase.compileThem
-    problems = collectProblems(testCase.outV1.jfile, testCase.outV2.jfile)
-    expected = readOracleFile(testCase.versionedFile("problems.txt").jfile)
+  def testCollectProblems(testCase: TestCase, direction: Direction): Try[Unit] = for {
+    () <- testCase.compileBoth
+    problems = collectProblems(direction.lhs(testCase).jfile, direction.rhs(testCase).jfile)
+    expected = readOracleFile(testCase.versionedFile(direction.oracleFile).jfile)
     () <- diffProblems(problems, expected)
   } yield ()
 

functional-tests/src/main/scala/com/typesafe/tools/mima/lib/Direction.scala

@@ -0,0 +1,18 @@
+package com.typesafe.tools.mima.lib
+
+sealed trait Direction {
+  def lhs(testCase: TestCase) = this match {
+    case Backwards => testCase.outV1
+    case Forwards  => testCase.outV2
+  }
+  def rhs(testCase: TestCase) = this match {
+    case Backwards => testCase.outV2
+    case Forwards  => testCase.outV1
+  }
+  def oracleFile = this match {
+    case Backwards => "problems.txt"
+    case Forwards  => "forwards.txt"
+  }
+}
+case object Backwards extends Direction
+case object  Forwards extends Direction

functional-tests/src/main/scala/com/typesafe/tools/mima/lib/TestCase.scala

@@ -1,6 +1,6 @@
 package com.typesafe.tools.mima.lib
 
-import java.io.{ ByteArrayOutputStream, File, PrintStream }
+import java.io.{ ByteArrayOutputStream, PrintStream }
 import java.net.{ URI, URLClassLoader }
 import javax.tools._
 
@@ -24,16 +24,14 @@ final class TestCase(val baseDir: Directory, val scalaCompiler: ScalaCompiler, v
   val outV2  = (baseDir / s"target/scala-$scalaBinaryVersion/v2-classes").toDirectory
   val outApp = (baseDir / s"target/scala-$scalaBinaryVersion/app-classes").toDirectory
 
-  lazy val compileThem: Try[Unit] = for {
-    () <- compileV1()
-    () <- compileV2()
-    () <- compileApp(outV1)
+  lazy val compileBoth: Try[Unit]   = for (() <- compileV1(); () <- compileV2()) yield ()
+  def compileV1(): Try[Unit]        = compileDir(srcV1, Nil, outV1)
+  def compileV2(): Try[Unit]        = compileDir(srcV2, Nil, outV2)
+  def compileApp(outLib: Directory) = for {
+    () <- compileBoth
+    () <- compileDir(srcApp, List(outLib), outApp)
   } yield ()
 
-  def compileV1()                   = compileDir(srcV1, Nil, outV1)
-  def compileV2()                   = compileDir(srcV2, Nil, outV2)
-  def compileApp(outLib: Directory) = compileDir(srcApp, List(outLib), outApp)
-
   def compileDir(srcDir: Directory, cp: List[Directory], out: Directory): Try[Unit] = for {
     () <- Try(recreateDir(out))
     () <- compileScala(srcDir, cp, out)

functional-tests/src/main/scala/com/typesafe/tools/mima/lib/TestCli.scala

@@ -16,7 +16,10 @@ object TestCli {
   val testsDir = Directory("functional-tests/src/test")
 
   def argsToTests(args: List[String], runTestCase: TestCase => Try[Unit]): Tests =
-    Tests(fromArgs(args).map(testCaseToTest1(_, runTestCase)))
+    testCasesToTests(fromArgs(args), runTestCase)
+
+  def testCasesToTests(testCases: List[TestCase], runTestCase: TestCase => Try[Unit]): Tests =
+    Tests(testCases.map(testCaseToTest1(_, runTestCase)))
 
   def testCaseToTest1(tc: TestCase, runTestCase: TestCase => Try[Unit]): Test1 =
     Test(s"${tc.scalaBinaryVersion} / ${tc.name}", runTestCase(tc))
@@ -25,7 +28,7 @@ object TestCli {
 
   @tailrec def postProcessConf(conf: Conf): Conf = conf match {
     case Conf(Nil, _) => postProcessConf(conf.copy(scalaVersions = List(hostScalaVersion)))
-    case Conf(_, Nil) => postProcessConf(conf.copy(dirs = allTestDirs()))
+    case Conf(_, Nil) => postProcessConf(conf.copy(dirs = allTestDirs().ensuring(_.nonEmpty)))
     case _            => conf
   }
 
@@ -40,7 +43,9 @@ object TestCli {
 
   def getJavaCompiler = ToolProvider.getSystemJavaCompiler
 
-  def allTestDirs() = testsDir.dirs.filter(_.files.exists(_.name == "problems.txt")).toList.sortBy(_.path)
+  def allTestDirs() = testsDir.dirs.filter(_.files.exists { f =>
+    f.name == Backwards.oracleFile || f.name == Forwards.oracleFile
+  }).toList.sortBy(_.path)
 
   final case class Conf(scalaVersions: List[String], dirs: List[Directory])
 

functional-tests/src/main/scala/com/typesafe/tools/mima/lib/UnitTests.scala

@@ -1,5 +1,6 @@
 package com.typesafe.tools.mima.lib
 
+import scala.reflect.io.Path
 import scala.util.{ Failure, Success, Try }
 
 import munit.GenericTest
@@ -9,17 +10,29 @@ object UnitTests {
   def munitTests(): List[GenericTest[Unit]] = TestCli.argsToTests(Nil, runTestCase).munitTests
 
   def runTestCase(testCase: TestCase): Try[Unit] = for {
-    () <- testNameCheck(testCase)
-    () <- CollectProblemsTest.testCollectProblems(testCase)
-    () <- AppRunTest.testAppRun(testCase)
+    () <- runTestCaseIf(testCase, Backwards)
+    () <- runTestCaseIf(testCase, Forwards)
   } yield ()
 
-  def testNameCheck(testCase: TestCase): Try[Unit] = {
-    val emptyProblemsTxt = testCase.blankFile(testCase.baseDir / "problems.txt")
-    testCase.baseDir.name.takeRight(4).dropWhile(_ != '-') match {
-      case "-ok"  => if (emptyProblemsTxt) Success(()) else Failure(new Exception("OK test with non-empty problems.txt"))
-      case "-nok" => if (emptyProblemsTxt) Failure(new Exception("NOK test with empty problems.txt")) else Success(())
-      case _      => Failure(new Exception("Missing '-ok' or '-nok' suffix in project name"))
+  def runTestCaseIf(testCase: TestCase, direction: Direction): Try[Unit] = {
+    if ((testCase.baseDir / direction.oracleFile).exists)
+      runTestCase1(testCase, direction)
+    else
+      Success(())
+  }
+
+  def runTestCase1(testCase: TestCase, direction: Direction): Try[Unit] = for {
+    () <- testNameCheck(testCase, direction.oracleFile)
+    () <- CollectProblemsTest.testCollectProblems(testCase, direction)
+    () <- AppRunTest.testAppRun(testCase, direction)
+  } yield ()
+
+  def testNameCheck(testCase: TestCase, oracleFile: Path): Try[Unit] = {
+    val emptyOracleFile = testCase.blankFile(testCase.baseDir / oracleFile)
+    testCase.name.takeRight(4).dropWhile(_ != '-') match {
+      case "-ok"  => if (emptyOracleFile) Success(()) else Failure(new Exception(s"OK test with non-empty ${oracleFile.name}"))
+      case "-nok" => if (emptyOracleFile) Failure(new Exception(s"NOK test with empty ${oracleFile.name}")) else Success(())
+      case _      => Failure(new Exception(s"Missing '-ok' or '-nok' suffix in project name: ${testCase.name}"))
     }
   }
 }