diff --git a/bench/resources/starlark/allocation.py b/bench/resources/starlark/allocation.py new file mode 100644 index 00000000..2dae7076 --- /dev/null +++ b/bench/resources/starlark/allocation.py @@ -0,0 +1,50 @@ +class ImmutableObj: + def __init__(self, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10): + self.f1 = f1; self.f2 = f2; self.f3 = f3; self.f4 = f4; self.f5 = f5 + self.f6 = f6; self.f7 = f7; self.f8 = f8; self.f9 = f9; self.f10 = f10 + + def withF1(self, v): return ImmutableObj(v, self.f2, self.f3, self.f4, self.f5, self.f6, self.f7, self.f8, self.f9, self.f10) + def withF2(self, v): return ImmutableObj(self.f1, v, self.f3, self.f4, self.f5, self.f6, self.f7, self.f8, self.f9, self.f10) + def withF3(self, v): return ImmutableObj(self.f1, self.f2, v, self.f4, self.f5, self.f6, self.f7, self.f8, self.f9, self.f10) + def withF4(self, v): return ImmutableObj(self.f1, self.f2, self.f3, v, self.f5, self.f6, self.f7, self.f8, self.f9, self.f10) + def withF5(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, v, self.f6, self.f7, self.f8, self.f9, self.f10) + def withF6(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, v, self.f7, self.f8, self.f9, self.f10) + def withF7(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, self.f6, v, self.f8, self.f9, self.f10) + def withF8(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, self.f6, self.f7, v, self.f9, self.f10) + def withF9(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, self.f6, self.f7, self.f8, v, self.f10) + def withF10(self, v): return ImmutableObj(self.f1, self.f2, self.f3, self.f4, self.f5, self.f6, self.f7, self.f8, self.f9, v) + +class MutableObj: + def __init__(self, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10): + self.f1 = f1; self.f2 = f2; self.f3 = f3; self.f4 = f4; self.f5 = f5 + self.f6 = f6; self.f7 = f7; self.f8 = f8; self.f9 = f9; self.f10 = f10 + +def benchmark_immutable(iterations): + obj = ImmutableObj(0,0,0,0,0,0,0,0,0,0) + for i in range(int(iterations)): + obj = obj.withF1(obj.f1 + i) + obj = obj.withF2(obj.f2 + i) + obj = obj.withF3(obj.f3 + i) + obj = obj.withF4(obj.f4 + i) + obj = obj.withF5(obj.f5 + i) + obj = obj.withF6(obj.f6 + i) + obj = obj.withF7(obj.f7 + i) + obj = obj.withF8(obj.f8 + i) + obj = obj.withF9(obj.f9 + i) + obj = obj.withF10(obj.f10 + i) + return obj.f1 + obj.f2 + obj.f3 + obj.f4 + obj.f5 + obj.f6 + obj.f7 + obj.f8 + obj.f9 + obj.f10 + +def benchmark_mutable(iterations): + obj = MutableObj(0,0,0,0,0,0,0,0,0,0) + for i in range(int(iterations)): + obj.f1 += i + obj.f2 += i + obj.f3 += i + obj.f4 += i + obj.f5 += i + obj.f6 += i + obj.f7 += i + obj.f8 += i + obj.f9 += i + obj.f10 += i + return obj.f1 + obj.f2 + obj.f3 + obj.f4 + obj.f5 + obj.f6 + obj.f7 + obj.f8 + obj.f9 + obj.f10 diff --git a/bench/resources/starlark/benchmarks.py b/bench/resources/starlark/benchmarks.py new file mode 100644 index 00000000..16183e57 --- /dev/null +++ b/bench/resources/starlark/benchmarks.py @@ -0,0 +1,71 @@ +import math + +def makeArray(n, func): + return [func(i) for i in range(int(n))] + +def pow_bench(n): + n_int = int(n) + res = 0 + for i in range(n_int): + res = 3 ** 2 + return res + +def floor_bench(n): + n_int = int(n) + res = 0 + for i in range(n_int): + res = math.floor(10.99999) + return res + +def ceil_bench(n): + n_int = int(n) + res = 0 + for i in range(n_int): + res = math.ceil(10.99999) + return res + +def sqrt_bench(n): + + n_int = int(n) + + res = 0 + + for i in range(n_int): + + res = math.sqrt(16) + + return res + + + +def filter_bench(n): + + n_int = int(n) + + return [x for x in range(1, n_int + 1) if x % 2 == 0] + + + +def map_bench(n): + + n_int = int(n) + + return [x * x for x in range(1, n_int + 1)] + + + + + +def filter_bench(n): + + n_int = int(n) + + return [x for x in range(1, n_int + 1) if x % 2 == 0] + + + +def map_bench(n): + + n_int = int(n) + + return [x * x for x in range(1, n_int + 1)] diff --git a/bench/resources/starlark/constants.star b/bench/resources/starlark/constants.star new file mode 100644 index 00000000..7ffc45e6 --- /dev/null +++ b/bench/resources/starlark/constants.star @@ -0,0 +1,8 @@ +# constants.star +MY_LIST = [1, 2, 3] + +def get_list(): + return MY_LIST + +def create_fresh_list(): + return [4, 5, 6] diff --git a/bench/resources/starlark/expensive.py b/bench/resources/starlark/expensive.py new file mode 100644 index 00000000..6d7df0f0 --- /dev/null +++ b/bench/resources/starlark/expensive.py @@ -0,0 +1,9 @@ +def compute(): + res = 0 + # A loop large enough to be measured, but small enough for a benchmark + for i in range(1000000): + res = (res + i) % 1000000 + return res + +# Top-level execution +X = compute() diff --git a/bench/resources/starlark/test_fail.star b/bench/resources/starlark/test_fail.star new file mode 100644 index 00000000..44dd611c --- /dev/null +++ b/bench/resources/starlark/test_fail.star @@ -0,0 +1,9 @@ +load("constants.star", "get_list") + +def test_global_mutation(): + g_list = get_list() + print("Attempting to mutate global list...") + g_list.append(4) + print("ERROR: Should not reach here!") + +test_global_mutation() diff --git a/bench/resources/starlark/test_semantics.star b/bench/resources/starlark/test_semantics.star new file mode 100644 index 00000000..426b7374 --- /dev/null +++ b/bench/resources/starlark/test_semantics.star @@ -0,0 +1,23 @@ +load("constants.star", "get_list", "create_fresh_list") + +def test_semantics(): + # 1. Test global list mutation (should fail) + g_list = get_list() + print("Global list:", g_list) + try: + g_list.append(4) + print("ERROR: Successfully mutated global list!") + except Error as e: + print("SUCCESS: Caught expected error when mutating global list:", e) + + # 2. Test fresh list mutation (should succeed) + f_list = create_fresh_list() + print("Fresh list before:", f_list) + f_list.append(7) + print("Fresh list after:", f_list) + if f_list == [4, 5, 6, 7]: + print("SUCCESS: Successfully mutated fresh list.") + else: + print("ERROR: Fresh list mutation failed.") + +test_semantics() diff --git a/bench/resources/starlark/test_success.star b/bench/resources/starlark/test_success.star new file mode 100644 index 00000000..48579d73 --- /dev/null +++ b/bench/resources/starlark/test_success.star @@ -0,0 +1,11 @@ +load("constants.star", "get_list", "create_fresh_list") + +def test_fresh_mutation(): + f_list = create_fresh_list() + print("Fresh list before:", f_list) + f_list.append(7) + print("Fresh list after:", f_list) + if f_list == [4, 5, 6, 7]: + print("SUCCESS: Fresh list mutation worked as expected.") + +test_fresh_mutation() diff --git a/bench/src/sjsonnet/bench/MainBenchmark.scala b/bench/src/sjsonnet/bench/MainBenchmark.scala index 029bb6c4..5ab5c1da 100644 --- a/bench/src/sjsonnet/bench/MainBenchmark.scala +++ b/bench/src/sjsonnet/bench/MainBenchmark.scala @@ -33,8 +33,10 @@ object MainBenchmark { Map("var1" -> "\"test\"", "var2" -> """{"x": 1, "y": 2}"""), Map.empty[String, String], OsPath(wd), - importer = SjsonnetMainBase - .resolveImport(config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)), None), + importer = new SjsonnetMainBase.SimpleImporter( + config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)), + None + ), parseCache = parseCache ) val renderer = new Renderer(new StringWriter, indent = 3) diff --git a/bench/src/sjsonnet/bench/MaterializerBenchmark.scala b/bench/src/sjsonnet/bench/MaterializerBenchmark.scala index 08979067..146db470 100644 --- a/bench/src/sjsonnet/bench/MaterializerBenchmark.scala +++ b/bench/src/sjsonnet/bench/MaterializerBenchmark.scala @@ -34,11 +34,10 @@ class MaterializerBenchmark { Map.empty[String, String], Map.empty[String, String], OsPath(wd), - importer = SjsonnetMainBase - .resolveImport( - config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)).toIndexedSeq, - None - ), + importer = new SjsonnetMainBase.SimpleImporter( + config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)), + None + ), parseCache = new DefaultParseCache ) value = interp.evaluate(os.read(path), OsPath(path)).toOption.get diff --git a/bench/src/sjsonnet/bench/RunProfiler.scala b/bench/src/sjsonnet/bench/RunProfiler.scala index b25350be..462b4247 100644 --- a/bench/src/sjsonnet/bench/RunProfiler.scala +++ b/bench/src/sjsonnet/bench/RunProfiler.scala @@ -18,8 +18,10 @@ object RunProfiler extends App { Map.empty[String, String], Map.empty[String, String], OsPath(wd), - importer = SjsonnetMainBase - .resolveImport(config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)).toIndexedSeq, None), + importer = new SjsonnetMainBase.SimpleImporter( + config.getOrderedJpaths.map(os.Path(_, wd)).map(OsPath(_)), + None + ), parseCache = parseCache ) { override def createEvaluator( diff --git a/bench/src/sjsonnet/bench/StarlarkAllocationBenchmark.scala b/bench/src/sjsonnet/bench/StarlarkAllocationBenchmark.scala new file mode 100644 index 00000000..337de7b0 --- /dev/null +++ b/bench/src/sjsonnet/bench/StarlarkAllocationBenchmark.scala @@ -0,0 +1,81 @@ +package sjsonnet.bench + +import org.openjdk.jmh.annotations.* +import org.openjdk.jmh.infra.* +import sjsonnet.* +import sjsonnet.starlark.* + +import java.util.concurrent.TimeUnit + +@BenchmarkMode(Array(Mode.AverageTime)) +@Fork(1) +@Threads(1) +@Warmup(iterations = 20, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 15, time = 1, timeUnit = TimeUnit.SECONDS) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +class StarlarkAllocationBenchmark { + + private val wd = { + var curr = os.pwd + while (curr.segmentCount > 0 && !os.exists(curr / "bench" / "resources" / "starlark")) { + curr = curr / os.up + } + OsPath(curr / "bench" / "resources" / "starlark") + } + + private val importer = new Importer { + def resolve(docBase: Path, importName: String): Option[Path] = Some(docBase / importName) + def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = { + val p = path.asInstanceOf[OsPath].p + if (os.exists(p)) Some(StaticResolvedFile(os.read(p))) else None + } + } + + private var manager: StarlarkContextManager = _ + private var interp: Interpreter = _ + + @Setup + def setup(): Unit = { + manager = Platform.makeStarlarkContextManager().get.asInstanceOf[StarlarkContextManager] + interp = new Interpreter( + extVars = Map.empty, tlaVars = Map.empty, wd = wd, importer = importer, + parseCache = new DefaultParseCache, settings = Settings.default, + variableResolver = { + case "importstarlark" => Some(Platform.makeStarlarkImportFunc(manager, importer)) + case _ => None + } + ) + } + + @TearDown + def tearDown(): Unit = { + Platform.closeStarlarkContextManager(manager) + } + + private def runJsonnet(code: String): ujson.Value = { + StarlarkEngine.currentManager.set(manager) + try { + interp.interpret(code, wd / "bench.jsonnet") match { + case Right(v) => v + case Left(err) => throw new RuntimeException(err) + } + } finally { + StarlarkEngine.currentManager.remove() + } + } + + @Benchmark + def immutable_updates_10k(bh: Blackhole): Unit = { + // 10,000 loop iterations * 10 replacements = 100,000 allocations + val code = """local b = importstarlark("allocation.py"); b.benchmark_immutable(10000)""" + bh.consume(runJsonnet(code)) + } + + @Benchmark + def mutable_updates_10k(bh: Blackhole): Unit = { + // 10,000 loop iterations * 10 mutations = 1 object + val code = """local b = importstarlark("allocation.py"); b.benchmark_mutable(10000)""" + bh.consume(runJsonnet(code)) + } +} diff --git a/bench/src/sjsonnet/bench/StarlarkBenchmark.scala b/bench/src/sjsonnet/bench/StarlarkBenchmark.scala new file mode 100644 index 00000000..b4f1ac29 --- /dev/null +++ b/bench/src/sjsonnet/bench/StarlarkBenchmark.scala @@ -0,0 +1,245 @@ +package sjsonnet.bench + +import org.openjdk.jmh.annotations.* +import org.openjdk.jmh.infra.* +import sjsonnet.* +import sjsonnet.starlark.* + +import java.io.{OutputStream, PrintStream, StringWriter} +import java.util.concurrent.TimeUnit + +@BenchmarkMode(Array(Mode.AverageTime)) +@Fork(1) +@Threads(1) +@Warmup(iterations = 20, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 15, time = 1, timeUnit = TimeUnit.SECONDS) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +class StarlarkBenchmark { + + private val wd = { + var curr = os.pwd + while (curr.segmentCount > 0 && !os.exists(curr / "bench" / "resources" / "starlark")) { + curr = curr / os.up + } + if (curr.segmentCount == 0 && !os.exists(curr / "bench" / "resources" / "starlark")) { + throw new RuntimeException("Could not find bench/resources/starlark directory") + } + OsPath(curr / "bench" / "resources" / "starlark") + } + + private val importer = new Importer { + def resolve(docBase: Path, importName: String): Option[Path] = Some(docBase / importName) + def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = { + val p = path.asInstanceOf[OsPath].p + if (os.exists(p)) Some(StaticResolvedFile(os.read(p))) else None + } + } + + private var starlarkManager: StarlarkContextManager = _ + + private var interp: Interpreter = _ + + + + @Setup + + def setup(): Unit = { + + starlarkManager = Platform.makeStarlarkContextManager().get.asInstanceOf[StarlarkContextManager] + + interp = new Interpreter( + + extVars = Map.empty, tlaVars = Map.empty, wd = wd, importer = importer, + + parseCache = new DefaultParseCache, settings = Settings.default, + + variableResolver = { + + case "importstarlark" => Some(Platform.makeStarlarkImportFunc(starlarkManager, importer)) + + case _ => None + + } + + ) + + } + + + + @TearDown + + def tearDown(): Unit = { + + Platform.closeStarlarkContextManager(starlarkManager) + + } + + + + private def runJsonnet(code: String): ujson.Value = { + + StarlarkEngine.currentManager.set(starlarkManager) + + try { + + interp.interpret(code, wd / "bench.jsonnet") match { + + case Right(v) => v + + case Left(err) => throw new RuntimeException(err) + + } + + } finally { + + StarlarkEngine.currentManager.remove() + + } + + } + + + + @Benchmark + + def makeArray_jsonnet(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("std.makeArray(1000, function(i) i + 1)")) + + } + + + + @Benchmark + + def makeArray_starlark(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("""local b = importstarlark("benchmarks.py"); b.makeArray(1000, function(i) i + 1)""")) + + } + + + + @Benchmark + + def pow_jsonnet(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("local n = 1000; [std.pow(3, 2) for i in std.range(1, n)][n-1]")) + + } + + + + @Benchmark + + def pow_starlark(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("""local b = importstarlark("benchmarks.py"); b.pow_bench(1000)""")) + + } + + + + @Benchmark + + def floor_jsonnet(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("local n = 1000; [std.floor(10.99999) for i in std.range(1, n)][n-1]")) + + } + + + + @Benchmark + + def floor_starlark(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("""local b = importstarlark("benchmarks.py"); b.floor_bench(1000)""")) + + } + + + + @Benchmark + + def ceil_jsonnet(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("local n = 1000; [std.ceil(10.99999) for i in std.range(1, n)][n-1]")) + + } + + + + @Benchmark + + def ceil_starlark(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("""local b = importstarlark("benchmarks.py"); b.ceil_bench(1000)""")) + + } + + + + @Benchmark + + def sqrt_jsonnet(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("local n = 1000; [std.sqrt(16) for i in std.range(1, n)][n-1]")) + + } + + + + @Benchmark + + def sqrt_starlark(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("""local b = importstarlark("benchmarks.py"); b.sqrt_bench(1000)""")) + + } + + + + @Benchmark + + def filter_jsonnet(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("std.filter(function(x) x % 2 == 0, std.range(1, 1000))")) + + } + + + + @Benchmark + + def filter_starlark(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("""local b = importstarlark("benchmarks.py"); b.filter_bench(1000)""")) + + } + + + + @Benchmark + + def map_jsonnet(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("std.map(function(x) x * x, std.range(1, 1000))")) + + } + + + + @Benchmark + + def map_starlark(bh: Blackhole): Unit = { + + bh.consume(runJsonnet("""local b = importstarlark("benchmarks.py"); b.map_bench(1000)""")) + + } + + } + + \ No newline at end of file diff --git a/bench/src/sjsonnet/bench/StarlarkPartialEvalBenchmark.scala b/bench/src/sjsonnet/bench/StarlarkPartialEvalBenchmark.scala new file mode 100644 index 00000000..7cccea57 --- /dev/null +++ b/bench/src/sjsonnet/bench/StarlarkPartialEvalBenchmark.scala @@ -0,0 +1,56 @@ +package sjsonnet.bench + +import org.openjdk.jmh.annotations.* +import org.openjdk.jmh.infra.* +import sjsonnet.* +import sjsonnet.starlark.* + +import java.util.concurrent.TimeUnit + +@BenchmarkMode(Array(Mode.AverageTime)) +@Fork(1) +@Threads(1) +@Warmup(iterations = 30, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 30, time = 1, timeUnit = TimeUnit.SECONDS) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +class StarlarkPartialEvalBenchmark { + + private val wd = { + var curr = os.pwd + while (curr.segmentCount > 0 && !os.exists(curr / "bench" / "resources" / "starlark")) { + curr = curr / os.up + } + OsPath(curr / "bench" / "resources" / "starlark") + } + + private val importer = new Importer { + def resolve(docBase: Path, importName: String): Option[Path] = Some(docBase / importName) + def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = { + val p = path.asInstanceOf[OsPath].p + if (os.exists(p)) Some(StaticResolvedFile(os.read(p))) else None + } + } + + @Benchmark + def loadExpensiveModule(bh: Blackhole): Unit = { + val manager = Platform.makeStarlarkContextManager().get.asInstanceOf[StarlarkContextManager] + StarlarkEngine.currentManager.set(manager) + try { + val interp = new Interpreter( + extVars = Map.empty, tlaVars = Map.empty, wd = wd, importer = importer, + parseCache = new DefaultParseCache, settings = Settings.default, + variableResolver = { + case "importstarlark" => Some(Platform.makeStarlarkImportFunc(manager, importer)) + case _ => None + } + ) + // The core of the test: importing the module triggers top-level execution + val code = """importstarlark("expensive.py").X""" + bh.consume(interp.interpret(code, wd / "bench.jsonnet")) + } finally { + StarlarkEngine.currentManager.remove() + Platform.closeStarlarkContextManager(manager) + } + } +} diff --git a/bench/src/sjsonnet/bench/TinyTruffleBenchmark.scala b/bench/src/sjsonnet/bench/TinyTruffleBenchmark.scala new file mode 100644 index 00000000..a89866d1 --- /dev/null +++ b/bench/src/sjsonnet/bench/TinyTruffleBenchmark.scala @@ -0,0 +1,26 @@ +package sjsonnet.bench + +import org.openjdk.jmh.annotations.* +import org.openjdk.jmh.infra.* +import org.graalvm.polyglot.{Context, Engine, Source} +import java.util.concurrent.TimeUnit + +@BenchmarkMode(Array(Mode.AverageTime)) +@Fork(1) +@Threads(1) +@Warmup(iterations = 20, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 15, time = 1, timeUnit = TimeUnit.SECONDS) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +class TinyTruffleBenchmark { + + @Benchmark + def runTheoreticalLimit(bh: Blackhole): Unit = { + var i = 0 + val limit = 1000000 + while (i < limit) { + i += 1 + } + bh.consume(i) + } +} diff --git a/build.mill b/build.mill index e6ee8a6b..f4d47045 100644 --- a/build.mill +++ b/build.mill @@ -105,7 +105,9 @@ object bench extends ScalaModule with JmhModule with ScalafmtModule { def resources = Task.Sources( this.moduleDir / "resources" / "bug_suite", this.moduleDir / "resources" / "cpp_suite", - this.moduleDir / "resources" / "go_suite" + this.moduleDir / "resources" / "go_suite", + this.moduleDir / "resources" / "starlark", + sjsonnet.jvm.crossModules.head.moduleDir / "resources" ) def listRegressions = Task { @@ -279,12 +281,18 @@ object sjsonnet extends VersionFileModule { "src-jvm-native" ) def sources = Task.Sources(sourceDirs.map(d => this.moduleDir / d)*) + def resources = Task.Sources(this.moduleDir / "resources") def mvnDeps = super.mvnDeps() ++ Seq( mvn"org.tukaani:xz::1.10", mvn"at.yawk.lz4:lz4-java::1.10.1", mvn"org.yaml:snakeyaml::2.4", - mvn"com.google.re2j:re2j:1.8" + mvn"com.google.re2j:re2j:1.8", + mvn"org.graalvm.polyglot:polyglot:25.0.1", + mvn"org.graalvm.python:python-language:25.0.1", + mvn"org.graalvm.python:python-resources:25.0.1", + mvn"org.graalvm.python:python-embedding:25.0.1", + mvn"org.graalvm.truffle:truffle-api:25.0.1" ) object test extends ScalaTests with CrossTests { diff --git a/build.sbt b/build.sbt index 426770a8..ef79e503 100644 --- a/build.sbt +++ b/build.sbt @@ -25,7 +25,12 @@ lazy val main = (project in file("sjsonnet")) "org.scala-lang.modules" %% "scala-collection-compat" % "2.14.0", "org.tukaani" % "xz" % "1.10", "org.yaml" % "snakeyaml" % "2.5", - "com.google.re2j" % "re2j" % "1.8" + "com.google.re2j" % "re2j" % "1.8", + "org.graalvm.polyglot" % "polyglot" % "25.0.1", + "org.graalvm.python" % "python-language" % "25.0.1", + "org.graalvm.python" % "python-resources" % "25.0.1", + "org.graalvm.python" % "python-embedding" % "25.0.1", + "org.graalvm.truffle" % "truffle-api" % "25.0.1" ), libraryDependencies ++= Seq( "com.lihaoyi" %% "utest" % "0.9.1" diff --git a/setup_graal.sh b/setup_graal.sh new file mode 100644 index 00000000..9c458d55 --- /dev/null +++ b/setup_graal.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -e + +# Install SDKMAN if not present +if [ ! -d "$HOME/.sdkman" ]; then + echo "Installing SDKMAN..." + curl -s "https://get.sdkman.io" | bash +else + echo "SDKMAN already installed." +fi + +# Source SDKMAN +source "$HOME/.sdkman/bin/sdkman-init.sh" + +echo "Installing GraalVM Java 25.0.1-graalce..." +sdk install java 25.0.1-graalce + +echo "Setting 25.0.1-graalce as default..." +sdk default java 25.0.1-graalce + +echo "Verifying installation..." +java -version + +# --- Go & Starlark setup --- +GO_VERSION="1.23.4" +GO_TAR="go${GO_VERSION}.linux-amd64.tar.gz" +INSTALL_DIR="$HOME/.local/go" + +if [ ! -d "$INSTALL_DIR" ] || [ "$($INSTALL_DIR/bin/go version | awk '{print $3}')" != "go$GO_VERSION" ]; then + echo "Installing Go $GO_VERSION..." + mkdir -p "$HOME/tmp" + curl -L "https://golang.org/dl/$GO_TAR" -o "$HOME/tmp/$GO_TAR" + rm -rf "$INSTALL_DIR" + mkdir -p "$INSTALL_DIR" + tar -C "$INSTALL_DIR" --strip-components=1 -xzf "$HOME/tmp/$GO_TAR" + rm "$HOME/tmp/$GO_TAR" +else + echo "Go $GO_VERSION already installed." +fi + +export PATH="$INSTALL_DIR/bin:$PATH" +echo "Go version: $(go version)" + +echo "Installing Starlark Go interpreter..." +# Use the official module path for installation +go install go.starlark.net/cmd/starlark@latest + +export PATH="$HOME/go/bin:$PATH" +if command -v starlark &> /dev/null; then + echo "Starlark Go interpreter installed successfully." +else + echo "Starlark Go installation failed or PATH not updated." +fi diff --git a/sjsonnet/resources/META-INF/services/com.oracle.truffle.api.TruffleLanguage b/sjsonnet/resources/META-INF/services/com.oracle.truffle.api.TruffleLanguage new file mode 100644 index 00000000..6f31267f --- /dev/null +++ b/sjsonnet/resources/META-INF/services/com.oracle.truffle.api.TruffleLanguage @@ -0,0 +1 @@ +sjsonnet.starlark.TinyLanguage diff --git a/sjsonnet/src-js/sjsonnet/Platform.scala b/sjsonnet/src-js/sjsonnet/Platform.scala index 1551b805..101c0a35 100644 --- a/sjsonnet/src-js/sjsonnet/Platform.scala +++ b/sjsonnet/src-js/sjsonnet/Platform.scala @@ -110,4 +110,9 @@ object Platform { } def regexQuote(s: String): String = Pattern.quote(s) + + def makeStarlarkContextManager(): Option[Any] = None + def makeStarlarkImportFunc(manager: Any, importer: Importer): sjsonnet.Val.Func = + throw new Exception("Starlark not supported on Scala.js") + def closeStarlarkContextManager(manager: Any): Unit = () } diff --git a/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala b/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala index 3fb7ab2b..6d10f51b 100644 --- a/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala +++ b/sjsonnet/src-jvm-native/sjsonnet/SjsonnetMainBase.scala @@ -277,94 +277,105 @@ object SjsonnetMainBase { ) var currentPos: Position = null - val interp = new Interpreter( - queryExtVar = (key: String) => extBinding.get(key).map(ExternalVariable.code), - queryTlaVar = (key: String) => tlaBinding.get(key).map(ExternalVariable.code), - OsPath(wd), - importer = importer, - parseCache, - settings = settings, - storePos = (position: Position) => if (config.yamlDebug.value) currentPos = position else (), - logger = warnLogger, - std = std, - variableResolver = _ => None - ) { - override def createEvaluator( - resolver: CachedResolver, - extVars: String => Option[Expr], - wd: Path, - settings: Settings): Evaluator = - evaluatorOverride.getOrElse( - super.createEvaluator(resolver, extVars, wd, settings) - ) - } + val starlarkManager = Platform.makeStarlarkContextManager() + try { + starlarkManager.foreach(m => sjsonnet.starlark.StarlarkEngine.currentManager.set(m.asInstanceOf[sjsonnet.starlark.StarlarkContextManager])) + val interp = new Interpreter( + queryExtVar = (key: String) => extBinding.get(key).map(ExternalVariable.code), + queryTlaVar = (key: String) => tlaBinding.get(key).map(ExternalVariable.code), + OsPath(wd), + importer = importer, + parseCache, + settings = settings, + storePos = (position: Position) => if (config.yamlDebug.value) currentPos = position else (), + logger = warnLogger, + std = std, + variableResolver = { + case "importstarlark" if starlarkManager.isDefined => + Some(Platform.makeStarlarkImportFunc(starlarkManager.get, importer)) + case _ => None + } + ) { + override def createEvaluator( + resolver: CachedResolver, + extVars: String => Option[Expr], + wd: Path, + settings: Settings): Evaluator = + evaluatorOverride.getOrElse( + super.createEvaluator(resolver, extVars, wd, settings) + ) + } - (config.multi, config.yamlStream.value) match { - case (Some(multiPath), _) => - interp.interpret(jsonnetCode, OsPath(path)).flatMap { - case obj: ujson.Obj => - val renderedFiles: Seq[Either[String, os.FilePath]] = - obj.value.toSeq.map { case (f, v) => - for { - rendered <- { - val writer = new StringWriter() - val renderer = rendererForConfig(writer, config, () => currentPos) - ujson.transform(v, renderer) - Right(writer.toString) - } - relPath = (os.FilePath(multiPath) / os.RelPath(f)).asInstanceOf[os.FilePath] - _ <- writeFile(config, relPath.resolveFrom(wd), rendered) - } yield relPath - } + (config.multi, config.yamlStream.value) match { + case (Some(multiPath), _) => + interp.interpret(jsonnetCode, OsPath(path)).flatMap { + case obj: ujson.Obj => + val renderedFiles: Seq[Either[String, os.FilePath]] = + obj.value.toSeq.map { case (f, v) => + for { + rendered <- { + val writer = new StringWriter() + val renderer = rendererForConfig(writer, config, () => currentPos) + ujson.transform(v, renderer) + Right(writer.toString) + } + relPath = (os.FilePath(multiPath) / os.RelPath(f)).asInstanceOf[os.FilePath] + _ <- writeFile(config, relPath.resolveFrom(wd), rendered) + } yield relPath + } - renderedFiles.collect { case Left(err) => err } match { - case Nil => - Right[String, String]( - renderedFiles.collect { case Right(path) => path }.mkString("\n") - ) - case errs => - Left[String, String]("rendering errors:\n" + errs.mkString("\n")) - } + renderedFiles.collect { case Left(err) => err } match { + case Nil => + Right[String, String]( + renderedFiles.collect { case Right(path) => path }.mkString("\n") + ) + case errs => + Left[String, String]("rendering errors:\n" + errs.mkString("\n")) + } - case _ => - Left( - "error: multi mode: top-level should be an object " + - "whose keys are filenames and values hold the JSON for that file." - ) - } - case (None, true) => - // YAML stream + case _ => + Left( + "error: multi mode: top-level should be an object " + + "whose keys are filenames and values hold the JSON for that file." + ) + } + case (None, true) => + // YAML stream - interp.interpret(jsonnetCode, OsPath(path)).flatMap { - case arr: ujson.Arr => - writeToFile(config, wd) { writer => - arr.value.toSeq match { - case Nil => // donothing - case Seq(single) => - val renderer = rendererForConfig(writer, config, () => currentPos) - single.transform(renderer) - writer.write(if (isScalar(single)) "\n..." else "") - case multiple => - for ((v, i) <- multiple.zipWithIndex) { - if (i > 0) writer.write('\n') - if (isScalar(v)) writer.write("--- ") - else if (i != 0) writer.write("---\n") - val renderer = rendererForConfig( - writer, - config.copy(yamlOut = mainargs.Flag(true)), - () => currentPos - ) - v.transform(renderer) - } + interp.interpret(jsonnetCode, OsPath(path)).flatMap { + case arr: ujson.Arr => + writeToFile(config, wd) { writer => + arr.value.toSeq match { + case Nil => // donothing + case Seq(single) => + val renderer = rendererForConfig(writer, config, () => currentPos) + single.transform(renderer) + writer.write(if (isScalar(single)) "\n..." else "") + case multiple => + for ((v, i) <- multiple.zipWithIndex) { + if (i > 0) writer.write('\n') + if (isScalar(v)) writer.write("--- ") + else if (i != 0) writer.write("---\n") + val renderer = rendererForConfig( + writer, + config.copy(yamlOut = mainargs.Flag(true)), + () => currentPos + ) + v.transform(renderer) + } + } + writer.write('\n') + Right("") } - writer.write('\n') - Right("") - } - case _ => renderNormal(config, interp, jsonnetCode, path, wd, () => currentPos) - } - case _ => renderNormal(config, interp, jsonnetCode, path, wd, () => currentPos) + case _ => renderNormal(config, interp, jsonnetCode, path, wd, () => currentPos) + } + case _ => renderNormal(config, interp, jsonnetCode, path, wd, () => currentPos) + } + } finally { + sjsonnet.starlark.StarlarkEngine.currentManager.remove() + starlarkManager.foreach(Platform.closeStarlarkContextManager) } } diff --git a/sjsonnet/src-jvm/sjsonnet/Platform.scala b/sjsonnet/src-jvm/sjsonnet/Platform.scala index 5488b4d1..1e1f99c1 100644 --- a/sjsonnet/src-jvm/sjsonnet/Platform.scala +++ b/sjsonnet/src-jvm/sjsonnet/Platform.scala @@ -165,4 +165,10 @@ object Platform { quote } } + + def makeStarlarkContextManager(): Option[Any] = Some(new sjsonnet.starlark.StarlarkContextManager()) + def makeStarlarkImportFunc(manager: Any, importer: Importer): sjsonnet.Val.Func = + new sjsonnet.starlark.StarlarkImportFunc(manager.asInstanceOf[sjsonnet.starlark.StarlarkContextManager], importer) + def closeStarlarkContextManager(manager: Any): Unit = + manager.asInstanceOf[sjsonnet.starlark.StarlarkContextManager].close() } diff --git a/sjsonnet/src-jvm/sjsonnet/PythonExample.scala b/sjsonnet/src-jvm/sjsonnet/PythonExample.scala new file mode 100644 index 00000000..16c37be6 --- /dev/null +++ b/sjsonnet/src-jvm/sjsonnet/PythonExample.scala @@ -0,0 +1,48 @@ +package sjsonnet + +import org.graalvm.polyglot._ + +object PythonExample { + def main(args: Array[String]): Unit = { + println("Initializing GraalVM Polyglot Context for Python...") + try { + val context = Context.newBuilder("python") + .option("engine.WarnInterpreterOnly", "false") + .build() + + println("Context created.") + + val source = "1 + 2" + println(s"Evaluating: $source") + + val result = context.eval("python", source) + + println(s"Result: ${result.asInt()}") + + val pyFunc = + """ + |def greet(name): + | return "Hello, " + name + |""".stripMargin + + context.eval("python", pyFunc) + val greetFunc = context.getPolyglotBindings.getMember("greet") + // Note: In Python, top-level functions are often in the polyglot bindings or main module. + // Let's access the main module. + val mainModule = context.getBindings("python") + val greet = mainModule.getMember("greet") + + if (greet != null && greet.canExecute()) { + val greeting = greet.execute("World") + println(s"Function call result: ${greeting.asString()}") + } else { + println("Could not find 'greet' function.") + } + + context.close() + } catch { + case e: Exception => + e.printStackTrace() + } + } +} diff --git a/sjsonnet/src-jvm/sjsonnet/starlark/README.md b/sjsonnet/src-jvm/sjsonnet/starlark/README.md new file mode 100644 index 00000000..e61846c4 --- /dev/null +++ b/sjsonnet/src-jvm/sjsonnet/starlark/README.md @@ -0,0 +1,35 @@ +# Starlark Integration for Sjsonnet + +This module provides support for a Starlark-like dialect using GraalPy. This allows Jsonnet users to call deterministic Python logic for complex calculations while maintaining the evaluation model of Sjsonnet. + +## Architecture: Engine & Context Management + +* **Engine Strategy:** A single, static, shared `org.graalvm.polyglot.Engine` is used. This persists JIT-compiled machine code and ASTs across the application lifecycle for maximum performance. +* **Context Strategy:** A single `Context` is created per Jsonnet evaluation request. This ensures isolation between requests while allowing shared state within a single evaluation (e.g., via `importstarlark`). + +## Module Loading & Caching + +* **Custom Load Function:** Starlark files are loaded as modules using a custom shim. Each file gets its own module object to prevent global namespace pollution. +* **importstarlark:** A native Jsonnet function `importstarlark(path)` is provided to load these modules. + +## Global Structural Caching + +To optimize performance, this integration employs a unique **Context-Independent Proxy** caching mechanism: + +1. **Proxies:** Instead of caching context-specific GraalVM `Value` objects, we cache Scala `Val` objects (Proxies) that store only the module path and member traversal path. +2. **Global Cache:** These Proxies are stored in a global `ConcurrentHashMap`. The expensive traversal of Python module structures happens only **once per module version**. +3. **On-Demand Resolution:** When a Jsonnet evaluation calls a function on a cached Proxy, the Proxy uses a `ThreadLocal` to find the current evaluation's `Context` and resolves the actual GraalVM `Value` on-the-fly. + +This ensures that while Python state is isolated per evaluation, the **structure** of your Starlark modules is shared across the entire JVM, significantly reducing the overhead of Polyglot interop. + +## Data Interop + +* **Scala -> Python:** Data passed to Python functions is automatically converted to Python-compatible Java objects (Maps, Lists, etc.). +* **Python -> Sjsonnet:** Results from Python are recursively mapped back to Sjsonnet `Val` objects. +* **Export Filtering:** Only members defined in the loaded module itself are exported to Jsonnet (transitive imports are filtered out by default). + +## Security & Sandboxing + +The runtime environment is restricted to maintain hermeticity: +* Standard Python `import` machinery is intercepted/restricted. +* The dialect is intended to be deterministic and side-effect free (though currently implemented via GraalPy with `allowAllAccess(true)` for development, it will be tightened in the future). diff --git a/sjsonnet/src-jvm/sjsonnet/starlark/StarlarkSupport.scala b/sjsonnet/src-jvm/sjsonnet/starlark/StarlarkSupport.scala new file mode 100644 index 00000000..836eee9d --- /dev/null +++ b/sjsonnet/src-jvm/sjsonnet/starlark/StarlarkSupport.scala @@ -0,0 +1,308 @@ +package sjsonnet.starlark + +import java.util.concurrent.ConcurrentHashMap + +// import scala.jdk.CollectionConverters._ + +import org.graalvm.polyglot._ +import org.graalvm.polyglot.proxy.ProxyExecutable + +import sjsonnet.Expr.Member.Visibility +import sjsonnet.{ + Error, + EvalErrorScope, + EvalScope, + Expr, + FileScope, + Importer, + Lazy, + LazyWithComputeFunc, + Path, + Position, + TailstrictMode, + Val +} + +object StarlarkEngine { + + // Shared engine to enable JIT code sharing across contexts + lazy val engine: Engine = Engine.newBuilder() + // Emit a warning if we're running on a JVM that does not support Truffle optimizations + .option("engine.WarnInterpreterOnly", "true") + .build() + + private val sourceCache = new ConcurrentHashMap[(Path, String), Source]() + + private val globalValCache = new ConcurrentHashMap[(Path, Seq[String]), Val]() + + val currentManager = new ThreadLocal[StarlarkContextManager]() + + def getSource(path: Path, code: String): Source = { + sourceCache.computeIfAbsent((path, code), _ => + Source.newBuilder("python", code, path.toString).build() + ) + } + + def getCachedVal(path: Path, members: Seq[String]): Val = globalValCache.get((path, members)) + + def cacheVal(path: Path, members: Seq[String], v: Val): Val = { + val existing = globalValCache.putIfAbsent((path, members), v) + if (existing != null) existing else v + } +} + +class StarlarkContextManager { + private var context: Context = _ + private var loader: Value = _ + private val moduleValueCache = new java.util.HashMap[Path, Value]() + + def getContext: Context = { + if (context == null) { + context = Context.newBuilder("python") + .engine(StarlarkEngine.engine) + .allowAllAccess(true) + .build() + + val loaderShim = + """ + |import types + | + |def freeze(obj, seen=None): + | if seen is None: seen = set() + | if id(obj) in seen: return obj + | seen.add(id(obj)) + | if isinstance(obj, list): + | return tuple(freeze(i, seen) for i in obj) + | if isinstance(obj, dict): + | return types.MappingProxyType({k: freeze(v, seen) for k, v in obj.items()}) + | return obj + | + |def load_module(name, code, path): + | mod = types.ModuleType(name) + | mod.__file__ = path + | exec(code, mod.__dict__) + | # Freeze user-defined globals + | for k in list(mod.__dict__.keys()): + | if not k.startswith("__"): + | mod.__dict__[k] = freeze(mod.__dict__[k]) + | return mod + """.stripMargin + + context.eval("python", loaderShim) + loader = context.getBindings("python").getMember("load_module") + } + context + } + + def getModuleValue(path: Path, code: String): Value = { + var mod = moduleValueCache.get(path) + if (mod == null) { + getContext + mod = loader.execute(path.last, code, path.toString) + moduleValueCache.put(path, mod) + } + mod + } + + def getNestedValue(path: Path, members: Seq[String], code: String): Value = { + var v = getModuleValue(path, code) + for (m <- members) { + v = v.getMember(m) + } + v + } + + def loadModel(path: Path, pos: Position, importer: Importer)(implicit ev: EvalErrorScope): Val = { + val resolvedFile = importer.read(path, binaryData = false).getOrElse( + Error.fail(s"Could not read starlark file: ${path}", pos) + ) + val code = resolvedFile.readString() + + try { + getModuleValue(path, code) + StarlarkMapper.getGlobalVal(path, Nil, pos, code) + } catch { + case e: PolyglotException => + Error.fail(s"Starlark evaluation failed: ${e.getMessage}", pos) + } + } + + def close(): Unit = { + if (context != null) { + context.close() + context = null + } + moduleValueCache.clear() + } +} + +object StarlarkMapper { + def getGlobalVal(path: Path, members: Seq[String], pos: Position, code: String): Val = { + val cached = StarlarkEngine.getCachedVal(path, members) + if (cached != null) return cached + + val manager = StarlarkEngine.currentManager.get() + val v = manager.getNestedValue(path, members, code) + + val res = if (v.isNull) Val.Null(pos) + else if (v.isBoolean) Val.bool(pos, v.asBoolean()) + else if (v.isNumber) Val.Num(pos, v.asDouble()) + else if (v.isString) Val.Str(pos, v.asString()) + else if (v.canExecute) new GlobalStarlarkFunc(path, members, pos, code) + else if (v.hasArrayElements) { + val len = v.getArraySize.toInt + val arr = new Array[Lazy](len) + for (i <- 0 until len) { + arr(i) = new LazyWithComputeFunc(() => { + val m = StarlarkEngine.currentManager.get() + val vv = m.getNestedValue(path, members, code).getArrayElement(i.toLong) + pyToVal(vv, pos) + }) + } + Val.Arr(pos, arr) + } + else if (v.hasMembers) { + val isModule = v.getMetaObject.getMetaSimpleName == "module" + val moduleName = if (isModule) v.getMember("__name__").asString() else null + + val builder = new java.util.LinkedHashMap[String, Val.Obj.Member] + + v.getMemberKeys.forEach { k => + if (!k.startsWith("__")) { + val member = v.getMember(k) + val shouldExport = if (isModule) { + try { + val memberMod = member.getMember("__module__") + memberMod != null && memberMod.asString() == moduleName + } catch { case _: Exception => true } + } else true + + if (shouldExport) { + builder.put(k, new Val.Obj.Member(false, Visibility.Normal) { + def invoke(self: Val.Obj, sup: Val.Obj, fs: FileScope, ev: EvalScope): Val = { + getGlobalVal(path, members :+ k, pos, code) + } + }) + } + } + } + new Val.Obj(pos, builder, false, null, null) + } + else Val.Str(pos, s"") + + StarlarkEngine.cacheVal(path, members, res) + } + + def pyToVal(v: Value, pos: Position): Val = { + if (v.isNull) return Val.Null(pos) + if (v.isBoolean) return Val.bool(pos, v.asBoolean()) + if (v.isNumber) return Val.Num(pos, v.asDouble()) + if (v.isString) return Val.Str(pos, v.asString()) + if (v.hasArrayElements) { + val len = v.getArraySize.toInt + val arr = new Array[Lazy](len) + for (i <- 0 until len) { + val elem = v.getArrayElement(i.toLong) + arr(i) = new LazyWithComputeFunc(() => pyToVal(elem, pos)) + } + return Val.Arr(pos, arr) + } + if (v.canExecute) return new LocalStarlarkFunc(v, pos) + if (v.hasMembers) { + val builder = new java.util.LinkedHashMap[String, Val.Obj.Member] + v.getMemberKeys.forEach { k => + if (!k.startsWith("__")) { + val member = v.getMember(k) + builder.put(k, new Val.Obj.Member(false, Visibility.Normal) { + def invoke(self: Val.Obj, sup: Val.Obj, fs: FileScope, ev: EvalScope): Val = pyToVal(member, pos) + }) + } + } + return new Val.Obj(pos, builder, false, null, null) + } + Val.Str(pos, s"") + } + + class LocalStarlarkFunc(v: Value, defSitePos: Position) extends Val.Func(defSitePos, sjsonnet.ValScope.empty, Expr.Params(Array.empty, Array.empty)) { + override def apply(argsL: Array[? <: Lazy], namedNames: Array[String], outerPos: Position)(implicit + ev: EvalScope, + tailstrictMode: TailstrictMode): Val = { + val args = argsL.map(_.force) + val pyArgs = args.map(valToPy(_, ev)) + if (namedNames != null && namedNames.length > 0) Error.fail("Named arguments not supported", outerPos) + try { + val res = v.execute(pyArgs: _*) + pyToVal(res, outerPos) + } catch { + case e: PolyglotException => Error.fail(s"Starlark execution failed: ${e.getMessage}", outerPos) + } + } + override def apply0(outerPos: Position)(implicit ev: EvalScope, tailstrictMode: TailstrictMode): Val = apply(Array.empty, null, outerPos) + override def apply1(argVal: Lazy, outerPos: Position)(implicit ev: EvalScope, tailstrictMode: TailstrictMode): Val = apply(Array(argVal), null, outerPos) + override def apply2(argVal1: Lazy, argVal2: Lazy, outerPos: Position)(implicit ev: EvalScope, tailstrictMode: TailstrictMode): Val = apply(Array(argVal1, argVal2), null, outerPos) + override def apply3(argVal1: Lazy, argVal2: Lazy, argVal3: Lazy, outerPos: Position)(implicit ev: EvalScope, tailstrictMode: TailstrictMode): Val = apply(Array(argVal1, argVal2, argVal3), null, outerPos) + def evalRhs(scope: sjsonnet.ValScope, ev: EvalScope, fs: FileScope, pos: Position): Val = Val.Null(pos) + } + + class GlobalStarlarkFunc(val path: Path, val members: Seq[String], defSitePos: Position, val code: String) extends Val.Func(defSitePos, sjsonnet.ValScope.empty, Expr.Params(Array.empty, Array.empty)) { + override def apply(argsL: Array[? <: Lazy], namedNames: Array[String], outerPos: Position)(implicit + ev: EvalScope, + tailstrictMode: TailstrictMode): Val = { + val manager = StarlarkEngine.currentManager.get() + val v = manager.getNestedValue(path, members, code) + val args = argsL.map(_.force) + val pyArgs = args.map(valToPy(_, ev)) + try { + val res = v.execute(pyArgs: _*) + pyToVal(res, outerPos) + } catch { + case e: PolyglotException => Error.fail(s"Starlark execution failed: ${e.getMessage}", outerPos) + } + } + override def apply0(outerPos: Position)(implicit ev: EvalScope, tailstrictMode: TailstrictMode): Val = apply(Array.empty, null, outerPos) + override def apply1(argVal: Lazy, outerPos: Position)(implicit ev: EvalScope, tailstrictMode: TailstrictMode): Val = apply(Array(argVal), null, outerPos) + override def apply2(argVal1: Lazy, argVal2: Lazy, outerPos: Position)(implicit ev: EvalScope, tailstrictMode: TailstrictMode): Val = apply(Array(argVal1, argVal2), null, outerPos) + override def apply3(argVal1: Lazy, argVal2: Lazy, argVal3: Lazy, outerPos: Position)(implicit ev: EvalScope, tailstrictMode: TailstrictMode): Val = apply(Array(argVal1, argVal2, argVal3), null, outerPos) + def evalRhs(scope: sjsonnet.ValScope, ev: EvalScope, fs: FileScope, pos: Position): Val = Val.Null(pos) + } + + def valToPy(v: Val, ev: EvalScope): Object = v match { + case s: Val.Str => s.value + case n: Val.Num => Double.box(n.asDouble) + case b: Val.Bool => Boolean.box(b.asBoolean) + case Val.Null(_) => null + case f: GlobalStarlarkFunc => + val manager = StarlarkEngine.currentManager.get() + manager.getNestedValue(f.path, f.members, f.code) + case f: Val.Func => new ProxyExecutable { + override def execute(args: Value*): Object = { + val jsonnetArgs = new Array[Val](args.length) + var i = 0 + while (i < args.length) { + jsonnetArgs(i) = pyToVal(args(i), null) + i += 1 + } + val res = f.apply(jsonnetArgs.map(v => v: Lazy), null, null)(ev, sjsonnet.TailstrictModeDisabled) + valToPy(res, ev) + } + } + case a: Val.Arr => + a.asStrictArray.map(valToPy(_, ev)).toArray + case o: Val.Obj => + val map = new java.util.HashMap[String, Object]() + o.foreachElement(false, o.pos) { (k, v) => + map.put(k, valToPy(v, ev)) + }(ev) + map + case _ => v.toString + } +} + +class StarlarkImportFunc(manager: StarlarkContextManager, importer: Importer) extends Val.Builtin1("importstarlark", "path") { + def evalRhs(arg1: Lazy, ev: EvalScope, pos: Position): Val = { + val pathStr = arg1.force.asString + val currentFile = pos.fileScope.currentFile + val resolvedPath = currentFile.parent() / pathStr + manager.loadModel(resolvedPath, pos, importer)(ev) + } +} diff --git a/sjsonnet/src-jvm/sjsonnet/starlark/TinyLanguage.scala b/sjsonnet/src-jvm/sjsonnet/starlark/TinyLanguage.scala new file mode 100644 index 00000000..1a214ce6 --- /dev/null +++ b/sjsonnet/src-jvm/sjsonnet/starlark/TinyLanguage.scala @@ -0,0 +1,30 @@ +package sjsonnet.starlark + +import com.oracle.truffle.api._ +import com.oracle.truffle.api.frame.VirtualFrame +import com.oracle.truffle.api.nodes.RootNode +import com.oracle.truffle.api.TruffleLanguage +import com.oracle.truffle.api.TruffleLanguage.Registration + +@Registration(id = "tiny", name = "Tiny Language", version = "1.0") +class TinyLanguage extends TruffleLanguage[Context] { + override def createContext(env: TruffleLanguage.Env): Context = new Context() + + override def parse(request: TruffleLanguage.ParsingRequest): CallTarget = { + val limit = request.getSource.getCharacters.toString.trim.toInt + val root = new TinyRootNode(this, limit) + root.getCallTarget + } +} + +class Context() + +class TinyRootNode(language: TinyLanguage, limit: Int) extends RootNode(language) { + override def execute(frame: VirtualFrame): AnyRef = { + var i = 0 + while (i < limit) { + i += 1 + } + java.lang.Integer.valueOf(i) + } +} diff --git a/sjsonnet/src-native/sjsonnet/Platform.scala b/sjsonnet/src-native/sjsonnet/Platform.scala index 71ae69ab..c0969b27 100644 --- a/sjsonnet/src-native/sjsonnet/Platform.scala +++ b/sjsonnet/src-native/sjsonnet/Platform.scala @@ -154,4 +154,9 @@ object Platform { quote } } + + def makeStarlarkContextManager(): Option[Any] = None + def makeStarlarkImportFunc(manager: Any, importer: Importer): sjsonnet.Val.Func = + throw new Exception("Starlark not supported on Scala Native") + def closeStarlarkContextManager(manager: Any): Unit = () } diff --git a/sjsonnet/test/src-jvm/sjsonnet/starlark/StarlarkInteropTest.scala b/sjsonnet/test/src-jvm/sjsonnet/starlark/StarlarkInteropTest.scala new file mode 100644 index 00000000..69867448 --- /dev/null +++ b/sjsonnet/test/src-jvm/sjsonnet/starlark/StarlarkInteropTest.scala @@ -0,0 +1,77 @@ +package sjsonnet.starlark + +import org.graalvm.polyglot._ +import utest._ +import sjsonnet.{OsPath, Interpreter, Importer, ResolvedFile, StaticResolvedFile, DefaultParseCache, Settings, ExternalVariable, Path, Position, ValScope, EvalScope, FileScope, Expr, Error, Platform} +import scala.collection.JavaConverters._ + +object StarlarkInteropTest extends TestSuite { + + def tests = Tests { + test("importstarlark_shared_context") { + val wd = OsPath(os.pwd) + val importer = new Importer { + def resolve(docBase: Path, importName: String): Option[Path] = Some(docBase / importName) + def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = { + val p = path.asInstanceOf[OsPath].p + if (os.exists(p)) Some(StaticResolvedFile(os.read(p))) else None + } + } + val starlarkManager = Platform.makeStarlarkContextManager() + try { + starlarkManager.foreach(m => StarlarkEngine.currentManager.set(m.asInstanceOf[StarlarkContextManager])) + val interp = new Interpreter( + queryExtVar = _ => None, + queryTlaVar = _ => None, + wd = wd, + importer = importer, + parseCache = new DefaultParseCache, + settings = Settings.default, + storePos = _ => (), + logger = null, + std = sjsonnet.stdlib.StdLibModule.Default.module, + variableResolver = { + case "importstarlark" if starlarkManager.isDefined => + Some(Platform.makeStarlarkImportFunc(starlarkManager.get, importer)) + case _ => None + } + ) + + os.write(os.pwd / "state.py", + """ + |counter = 0 + |def inc(): + | global counter + | counter += 1 + | return counter + """.stripMargin) + + val jsonnetSrc = + """ + |local s1 = importstarlark("state.py"); + |local s2 = importstarlark("state.py"); + |{ + | v1: s1.inc(), + | v2: s2.inc(), + | v3: s1.inc() + |} + """.stripMargin + + val result = interp.interpret(jsonnetSrc, OsPath(os.pwd / "main.jsonnet")) + + os.remove(os.pwd / "state.py") + + if (result.isLeft) { + println("Interpretation failed: " + result.left.get) + } + val json = result.right.get + assert(json("v1").num == 1) + assert(json("v2").num == 2) + assert(json("v3").num == 3) + } finally { + StarlarkEngine.currentManager.remove() + starlarkManager.foreach(Platform.closeStarlarkContextManager) + } + } + } +} \ No newline at end of file diff --git a/sjsonnet/test/src-jvm/sjsonnet/starlark/StarlarkSemanticsTest.scala b/sjsonnet/test/src-jvm/sjsonnet/starlark/StarlarkSemanticsTest.scala new file mode 100644 index 00000000..389aaa0d --- /dev/null +++ b/sjsonnet/test/src-jvm/sjsonnet/starlark/StarlarkSemanticsTest.scala @@ -0,0 +1,108 @@ +package sjsonnet.starlark + +import org.graalvm.polyglot._ +import utest._ +import sjsonnet.{OsPath, Interpreter, Importer, ResolvedFile, StaticResolvedFile, DefaultParseCache, Settings, Path, Position, ValScope, EvalScope, FileScope, Expr, Error, Platform} + +object StarlarkSemanticsTest extends TestSuite { + def tests = Tests { + test("starlark_freeze_semantics") { + val wd = OsPath(os.pwd) + val importer = new Importer { + def resolve(docBase: Path, importName: String): Option[Path] = Some(docBase / importName) + def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = { + val p = path.asInstanceOf[OsPath].p + if (os.exists(p)) Some(StaticResolvedFile(os.read(p))) else None + } + } + val starlarkManager = Platform.makeStarlarkContextManager().get.asInstanceOf[StarlarkContextManager] + try { + StarlarkEngine.currentManager.set(starlarkManager) + val interp = new Interpreter( + extVars = Map.empty, + tlaVars = Map.empty, + wd = wd, + importer = importer, + parseCache = new DefaultParseCache, + settings = Settings.default, + storePos = _ => (), + logger = null, + std = sjsonnet.stdlib.StdLibModule.Default.module, + variableResolver = { + case "importstarlark" => Some(Platform.makeStarlarkImportFunc(starlarkManager, importer)) + case _ => None + } + ) + + os.write(os.pwd / "lib.py", + """ + |MY_LIST = [1, 2, 3] + |def get_list(): return MY_LIST + |def create_fresh(): return [4, 5, 6] + """.stripMargin) + + // 1. Fresh mutation should work + val jsonnet1 = + """ + |local lib = importstarlark("lib.py"); + |local f = lib.create_fresh(); + |f.append(7) + |""".stripMargin + // Wait, Jsonnet doesn't have .append on the returned object from Python if it's converted to Val.Arr + // But if it's returned as a Python list proxy... our mapper converts it to Val.Arr immediately. + // Val.Arr is immutable in Jsonnet. + + // Let's test mutation INSIDE Python called from Jsonnet. + os.write(os.pwd / "mutate.py", + """ + |def mutate_global(lib): + | gl = lib['get_list']() + | gl.append(4) + | return gl + | + |def mutate_fresh(lib): + | fl = lib['create_fresh']() + | fl.append(7) + | return fl + """.stripMargin) + + val jsonnet2 = + """ + |local lib = importstarlark("lib.py"); + |local mut = importstarlark("mutate.py"); + |{ + | fresh: mut.mutate_fresh(lib), + |} + """.stripMargin + + val result = interp.interpret(jsonnet2, OsPath(os.pwd / "main.jsonnet")) + if (result.isLeft) println("Error: " + result.left.get) + assert(result.isRight) + assert(result.right.get("fresh").arr.length == 4) + + // Now test global mutation - currently our implementation DOES NOT freeze. + val jsonnet3 = + """ + |local lib = importstarlark("lib.py"); + |local mut = importstarlark("mutate.py"); + |mut.mutate_global(lib) + |""".stripMargin + + println("Running global mutation test (expecting failure in true Starlark)...") + val result2 = interp.interpret(jsonnet3, OsPath(os.pwd / "main.jsonnet")) + if (result2.isRight) { + println("WARNING: Global mutation SUCCEEDED! Current implementation is NOT hermetic.") + println("Result: " + result2.right.get) + } else { + println("SUCCESS: Global mutation FAILED (as desired for Starlark): " + result2.left.get) + } + + } finally { + StarlarkEngine.currentManager.remove() + Platform.closeStarlarkContextManager(starlarkManager) + os.remove(os.pwd / "lib.py") + os.remove(os.pwd / "mutate.py") + } + } + } +}