OpenAPI generator with support for optional aws-api-gateway extensions

.github/workflows/ci.yml

@@ -24,5 +24,7 @@ jobs:
         distribution: 'zulu'
     - name: print Java version
       run: java -version
+    - name: Set up sbt
+      uses: sbt/setup-sbt@v1
     - name: Build
       run: sbt ++${{ matrix.scala }} clean test

build.sbt

@@ -6,6 +6,10 @@ ThisBuild / scalaVersion := "2.13.11"
 
 ThisBuild / javacOptions ++= Seq("-source", "17", "-target", "17")
 
+ThisBuild / libraryDependencySchemes ++= Seq(
+  "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always,
+)
+
 lazy val allScalacOptions = Seq(
   "-deprecation",
   "-feature",
@@ -48,8 +52,8 @@ lazy val lib = project
 
 lazy val generator = project
   .in(file("generator"))
-  .dependsOn(csharpGenerator, scalaGenerator, rubyGenerator, javaGenerator, goGenerator, androidGenerator, kotlinGenerator, graphQLGenerator, javaAwsLambdaPojos, postmanGenerator, csvGenerator)
-  .aggregate(csharpGenerator, scalaGenerator, rubyGenerator, javaGenerator, goGenerator, androidGenerator, kotlinGenerator, graphQLGenerator, javaAwsLambdaPojos, postmanGenerator, csvGenerator)
+  .dependsOn(csharpGenerator, scalaGenerator, rubyGenerator, javaGenerator, goGenerator, androidGenerator, kotlinGenerator, graphQLGenerator, javaAwsLambdaPojos, postmanGenerator, csvGenerator, openapiGenerator)
+  .aggregate(csharpGenerator, scalaGenerator, rubyGenerator, javaGenerator, goGenerator, androidGenerator, kotlinGenerator, graphQLGenerator, javaAwsLambdaPojos, postmanGenerator, csvGenerator, openapiGenerator)
   .enablePlugins(PlayScala)
   .enablePlugins(JavaAgent)
   .settings(commonSettings: _*)
@@ -187,6 +191,22 @@ lazy val postmanGenerator = project
     )
   )
 
+lazy val openapiGenerator = project
+  .in(file("openapi-generator"))
+  .dependsOn(lib, lib % "test->test")
+  .settings(commonSettings: _*)
+  .settings(resolversSettings)
+  .settings(
+    resolvers += "jitpack" at "https://jitpack.io",
+    libraryDependencies ++= Seq(
+      "com.softwaremill.sttp.apispec" %% "openapi-model" % "0.11.10",
+      "com.softwaremill.sttp.apispec" %% "openapi-circe" % "0.11.10",
+      "com.softwaremill.sttp.apispec" %% "openapi-circe-yaml" % "0.11.10",
+      "io.circe" %% "circe-generic-extras" % "0.14.4",
+      "com.github.apicollective" %% "apibuilder-validation" % "0.5.8",
+    ),
+  )
+
 lazy val commonSettings: Seq[Setting[_]] = Seq(
   name ~= ("apibuilder-generator-" + _),
   organization := "io.apibuilder",

generator/app/controllers/Generators.scala

@@ -598,5 +598,27 @@ object Generators {
       status = lib.generator.Status.InDevelopment,
       codeGenerator = Some(generator.csharp.CSharpGenerator)
     ),
+    CodeGenTarget(
+      metaData = Generator(
+        key = "openapi_json",
+        name = "OpenAPI JSON",
+        description = Some("Generates OpenAPI 3.0.3 specification in JSON format from an API Builder service. Imported models are inlined into the output."),
+        language = Some("json"),
+        attributes = Seq("aws_api_gateway"),
+      ),
+      status = lib.generator.Status.Alpha,
+      codeGenerator = Some(generator.openapi.OpenApiJsonGenerator),
+    ),
+    CodeGenTarget(
+      metaData = Generator(
+        key = "openapi_yaml",
+        name = "OpenAPI YAML",
+        description = Some("Generates OpenAPI 3.0.3 specification in YAML format from an API Builder service. Imported models are inlined into the output."),
+        language = Some("yaml"),
+        attributes = Seq("aws_api_gateway"),
+      ),
+      status = lib.generator.Status.Alpha,
+      codeGenerator = Some(generator.openapi.OpenApiYamlGenerator),
+    ),
   ).sortBy(_.metaData.key)
 }

openapi-generator/src/main/scala/generator/openapi/AwsExtension.scala

@@ -0,0 +1,22 @@
+package generator.openapi
+
+import generator.openapi.aws.{AwsApiGatewayConfig, AwsApiGatewayEnricher}
+import io.apibuilder.generator.v0.models.InvocationForm
+import io.apibuilder.spec.v0.models.Service
+import sttp.apispec.openapi.OpenAPI
+
+object AwsExtension {
+
+  private val AttributeName = "aws_api_gateway"
+
+  def maybeEnrich(openApi: OpenAPI, form: InvocationForm, service: Service): Either[Seq[String], OpenAPI] = {
+    form.attributes.find(_.name == AttributeName) match {
+      case None => Right(openApi)
+      case Some(attr) =>
+        AwsApiGatewayConfig.fromJson(attr.value) match {
+          case Left(err) => Left(Seq(s"$AttributeName: $err"))
+          case Right(config) => Right(AwsApiGatewayEnricher.enrich(openApi, config, service))
+        }
+    }
+  }
+}

openapi-generator/src/main/scala/generator/openapi/HeaderConverter.scala

@@ -0,0 +1,51 @@
+package generator.openapi
+
+import io.apibuilder.spec.v0.{models => ab}
+import sttp.apispec.SecurityScheme
+
+import scala.collection.immutable.ListMap
+
+object HeaderConverter {
+
+  case class Result(
+    securitySchemes: ListMap[String, Either[sttp.apispec.openapi.Reference, SecurityScheme]],
+    globalSecurity: List[ListMap[String, Vector[String]]],
+  )
+
+  def convert(headers: Seq[ab.Header]): Result = {
+    if (headers.isEmpty) return Result(ListMap.empty, Nil)
+
+    val schemes = headers.map { header =>
+      val schemeName = schemeNameFor(header.name)
+      val scheme = if (header.name.equalsIgnoreCase("Authorization")) {
+        SecurityScheme(
+          `type` = "http",
+          scheme = Some("bearer"),
+          description = header.description,
+        )
+      } else {
+        SecurityScheme(
+          `type` = "apiKey",
+          name = Some(header.name),
+          in = Some("header"),
+          description = header.description,
+        )
+      }
+      schemeName -> Right(scheme)
+    }
+
+    val security = headers.map { header =>
+      ListMap(schemeNameFor(header.name) -> Vector.empty[String])
+    }
+
+    Result(
+      securitySchemes = ListMap.from(schemes),
+      globalSecurity = security.toList,
+    )
+  }
+
+  private def schemeNameFor(headerName: String): String = {
+    if (headerName.equalsIgnoreCase("Authorization")) "bearerAuth"
+    else headerName.toLowerCase.replaceAll("[^a-z0-9]+", "_").replaceAll("^_|_$", "") + "_header"
+  }
+}

openapi-generator/src/main/scala/generator/openapi/MultiServiceExtractor.scala

@@ -0,0 +1,108 @@
+package generator.openapi
+
+import io.apibuilder.spec.v0.models.Service
+import io.apibuilder.validation.{ApiBuilderType, MultiService}
+
+import scala.annotation.tailrec
+
+case class MultiServiceExtractor(
+  sourceMultiService: MultiService,
+) {
+
+  private val ListRx = "^\\[(.*)\\]$".r
+  private val MapRx = "^map\\[(.*)\\]$".r
+
+  def extractReferencedTypes(service: Service): MultiService = {
+    val ns = service.namespace
+
+    val definedTypes = service.models.map(_.name) ++
+      service.enums.map(_.name) ++
+      service.unions.map(_.name) ++
+      service.interfaces.map(_.name)
+
+    val fieldTypes = service.models.flatMap(_.fields.map(_.`type`))
+    val interfaceFieldTypes = service.interfaces.flatMap(_.fields.map(_.`type`))
+    val unionMemberTypes = service.unions.flatMap(_.types.map(_.`type`))
+    val resourceTypes = service.resources.flatMap { r =>
+      Seq(r.`type`) ++ r.operations.flatMap { op =>
+        op.body.map(_.`type`).toSeq ++
+          op.responses.map(_.`type`) ++
+          op.parameters.map(_.`type`)
+      }
+    }
+
+    val allRefs = (definedTypes ++ fieldTypes ++ interfaceFieldTypes ++ unionMemberTypes ++ resourceTypes)
+      .map(stripWrapper)
+      .distinct
+
+    val seeds = allRefs.flatMap { ref =>
+      sourceMultiService.findType(ns, ref).collect { case t: ApiBuilderType => t }
+    }.toList
+
+    val types = collectAll(seeds, Set.empty)
+    buildMultiService(types)
+  }
+
+  private def childTypeRefs(t: ApiBuilderType): Seq[(String, String)] = t match {
+    case m: ApiBuilderType.Model =>
+      m.model.fields.map(f => stripWrapper(f.`type`) -> m.namespace)
+    case u: ApiBuilderType.Union =>
+      u.union.types.map(ut => ut.`type` -> u.namespace)
+    case i: ApiBuilderType.Interface =>
+      i.interface.fields.map(f => stripWrapper(f.`type`) -> i.namespace)
+    case _: ApiBuilderType.Enum => Nil
+  }
+
+  @tailrec
+  private def collectAll(pending: List[ApiBuilderType], acc: Set[ApiBuilderType]): Set[ApiBuilderType] = {
+    pending match {
+      case Nil => acc
+      case head :: tail if acc.exists(_.qualified == head.qualified) =>
+        collectAll(tail, acc)
+      case head :: tail =>
+        val children = childTypeRefs(head).flatMap { case (name, ns) =>
+          sourceMultiService.findType(ns, name).collect { case t: ApiBuilderType => t }
+        }
+        collectAll(children.toList ::: tail, acc + head)
+    }
+  }
+
+  @tailrec
+  private def stripWrapper(typeName: String): String = {
+    typeName match {
+      case ListRx(inner) => stripWrapper(inner)
+      case MapRx(inner) => stripWrapper(inner)
+      case other => other
+    }
+  }
+
+  private def buildMultiService(types: Set[ApiBuilderType]): MultiService = {
+    import io.apibuilder.validation.ApiBuilderService
+
+    val services = types
+      .groupBy(_.service.namespace)
+      .toList
+      .map { case (_, nsTypes) =>
+        val base = nsTypes.head.service.service.copy(
+          models = Nil,
+          enums = Nil,
+          interfaces = Nil,
+          unions = Nil,
+          resources = Nil,
+          headers = Nil,
+          imports = Nil,
+        )
+        val service = nsTypes.foldLeft(base) { (svc, t) =>
+          t match {
+            case m: ApiBuilderType.Model => svc.copy(models = m.model +: svc.models)
+            case e: ApiBuilderType.Enum => svc.copy(enums = e.`enum` +: svc.enums)
+            case i: ApiBuilderType.Interface => svc.copy(interfaces = i.interface +: svc.interfaces)
+            case u: ApiBuilderType.Union => svc.copy(unions = u.union +: svc.unions)
+          }
+        }
+        ApiBuilderService(service)
+      }
+
+    MultiService(services)
+  }
+}

openapi-generator/src/main/scala/generator/openapi/OpenApiConverter.scala

@@ -0,0 +1,56 @@
+package generator.openapi
+
+import io.apibuilder.spec.v0.models.Service
+import io.apibuilder.validation.{ApiBuilderService, MultiService}
+import sttp.apispec.openapi._
+
+object OpenApiConverter {
+
+  def convert(service: Service, importedServices: Seq[Service]): OpenAPI = {
+    val allServices = (Seq(service) ++ importedServices).map(ApiBuilderService.apply).toList
+    val multiService = MultiService(allServices)
+    buildOpenApi(service, multiService)
+  }
+
+  private def buildOpenApi(service: Service, multiService: MultiService): OpenAPI = {
+    val primaryAbs = ApiBuilderService(service)
+    val combined = if (multiService.services.exists(_.namespace == service.namespace)) {
+      multiService
+    } else {
+      MultiService(primaryAbs :: multiService.services)
+    }
+    val filtered = MultiServiceExtractor(combined).extractReferencedTypes(service)
+
+    val resolver = new TypeResolver(filtered)
+
+    val schemas = SchemaConverter.convert(filtered, resolver)
+    val paths = PathConverter.convert(service.resources, resolver)
+    val headerResult = HeaderConverter.convert(service.headers)
+
+    val components = if (schemas.nonEmpty || headerResult.securitySchemes.nonEmpty) {
+      Some(
+        Components(
+          schemas = schemas,
+          securitySchemes = headerResult.securitySchemes,
+        ),
+      )
+    } else {
+      None
+    }
+
+    val servers = service.baseUrl.map(url => Server(url = url)).toList
+
+    OpenAPI(
+      openapi = "3.0.3",
+      info = Info(
+        title = service.name,
+        version = service.version,
+        description = service.description,
+      ),
+      servers = servers,
+      paths = paths,
+      components = components,
+      security = if (headerResult.globalSecurity.nonEmpty) headerResult.globalSecurity else Nil,
+    )
+  }
+}

openapi-generator/src/main/scala/generator/openapi/OpenApiJsonGenerator.scala

@@ -0,0 +1,29 @@
+package generator.openapi
+
+import generator.ServiceFileNames
+import io.apibuilder.generator.v0.models.{File, InvocationForm}
+import lib.generator.CodeGenerator
+
+object OpenApiJsonGenerator extends CodeGenerator {
+
+  override def invoke(form: InvocationForm): Either[Seq[String], Seq[File]] = {
+    val service = form.service
+    val importedServices = form.importedServices.getOrElse(Nil)
+    val openApi = OpenApiConverter.convert(service, importedServices)
+
+    AwsExtension.maybeEnrich(openApi, form, service).map { enrichedOpenApi =>
+      val json = OutputWriter.toJson(enrichedOpenApi, pretty = true)
+      Seq(
+        ServiceFileNames.toFile(
+          namespace = service.namespace,
+          organizationKey = service.organization.key,
+          applicationKey = service.application.key,
+          version = service.version,
+          suffix = "OpenApi",
+          contents = json,
+          languages = Some("json"),
+        )
+      )
+    }
+  }
+}

openapi-generator/src/main/scala/generator/openapi/OpenApiYamlGenerator.scala

@@ -0,0 +1,29 @@
+package generator.openapi
+
+import generator.ServiceFileNames
+import io.apibuilder.generator.v0.models.{File, InvocationForm}
+import lib.generator.CodeGenerator
+
+object OpenApiYamlGenerator extends CodeGenerator {
+
+  override def invoke(form: InvocationForm): Either[Seq[String], Seq[File]] = {
+    val service = form.service
+    val importedServices = form.importedServices.getOrElse(Nil)
+    val openApi = OpenApiConverter.convert(service, importedServices)
+
+    AwsExtension.maybeEnrich(openApi, form, service).map { enrichedOpenApi =>
+      val yaml = OutputWriter.toYaml(enrichedOpenApi)
+      Seq(
+        ServiceFileNames.toFile(
+          namespace = service.namespace,
+          organizationKey = service.organization.key,
+          applicationKey = service.application.key,
+          version = service.version,
+          suffix = "OpenApi",
+          contents = yaml,
+          languages = Some("text"),
+        )
+      )
+    }
+  }
+}