Working demonstration of imports for C++. Checkpointing.

Author: pcj

bzl/base/class.bzl

@@ -33,9 +33,9 @@ def build_generated_files(lang, self):
         fail("Empty proto file input list.")
 
     for srcfile in self.get("srcs", []):
-        basename = srcfile.basename[:-len(".proto")]
+        base = srcfile.basename[:-len(".proto")]
         for ext in exts:
-            pbfile = ctx.new_file(srcfile, basename + ext)
+            pbfile = ctx.new_file(base + ext)
             self["provides"] += [pbfile]
 
 
@@ -54,11 +54,42 @@ def build_generated_filenames(lang, self):
             self["outs"] += [srcfile.rsplit('.', 1)[0] + ext]
 
 
-def build_imports(lang, self):
+def build_paths(lang, self):
     """Build the list of imports"""
     ctx = self["ctx"]
     if ctx:
-        self["imports"] = self.get("imports", []) + ctx.attr.imports
+        self["paths"] = self.get("paths", []) + ctx.attr.paths
+
+def build_imports(lang, self):
+    """Copy any named proto files in the imports to the genfiles area to make visible to protoc.  Not sure this is necessary anymore."""
+    ctx = self["ctx"]
+    if not ctx:
+        fail("Bazel context is required for build_imports")
+    #print("ctx.attr.imports: %s" % type(ctx.attr.imports))
+    for target in ctx.attr.imports:
+        for srcfile in target.files:
+            print("srcfile: %s" % srcfile.path)
+            dstfile = ctx.new_file(srcfile.path)
+            #dstfile = ctx.new_file(srcfile.path)
+            # By declaring that the proto action (happening later)
+            # depends on these files, we ensure that the
+            # CpImportToPackageGenfile action is run.  Without this
+            # linkage, bazel assumes the action isn't required.
+            self["requires"] += [dstfile]
+            if ctx.attr.verbose > 1:
+                print("Copying import %s --> %s" % (srcfile.path, dstfile.path))
+            ctx.action(mnemonic = "CpImportToPackageGenfiles",
+                       inputs = [srcfile],
+                       outputs = [dstfile],
+                       arguments = [srcfile.path, dstfile.path],
+                       command = "cp $1 $2")
+            # These probably need to be compiled also.  Nope: if you
+            # try this you get "inconsistent package names" from
+            # https://github.com/golang/protobuf/blob/1687f003bf0280bd6182a5a1c7241856b269a6c0/protoc-gen-go/generator/generator.go#L750
+            #self["srcs"] += [dstfile]
+
+            #self["go_plugin_options"] = self.get("go_plugin_options", []) + ["M" + srcfile.path + "="]
+    self["paths"] += [self["outdir"]]
 
 def build_plugin_out(name, key, lang, self):
     #print("build_plugin_out(%s, %s)" % (name, key))
@@ -67,7 +98,7 @@ def build_plugin_out(name, key, lang, self):
     plugin = getattr(lang, key)
     opts = getattr(plugin, "default_options", [])
     opts += self.get(key + "_plugin_options", [])
-    outdir = self["gendir"]
+    outdir = self["outdir"]
     if opts:
         outdir = ",".join(opts) + ":" + outdir
     self["args"] += ["--%s_out=%s" % (name, outdir)]
@@ -133,7 +164,7 @@ def build_protobuf_invocation(lang, self):
 def build_protoc_command(lang, self):
     """Build a command list required for genrule execution"""
     self["cmd"] += ["$(location %s)" % self["protoc"]]
-    self["cmd"] += ["-I" + i for i in self["imports"]]
+    self["cmd"] += ["--proto_path=" + i for i in self["paths"]]
     self["cmd"] += self["args"]
     self["cmd"] += ["$(location " + proto + ")" for proto in self["protos"]]
 
@@ -188,6 +219,7 @@ CLASS = struct(
     build_generated_filenames = build_generated_filenames,
     build_imports = build_imports,
     build_inputs = build_inputs,
+    build_paths = build_paths,
     build_tools = build_tools,
     build_protobuf_invocation = build_protobuf_invocation,
     build_protobuf_out = build_protobuf_out,

bzl/cpp/rules.bzl

@@ -7,12 +7,13 @@ cc_proto_compile = implement(["cpp"])
 def cc_proto_library(
     name,
     protos,
-    copy_protos_to_genfiles = True,
+    copy_protos_to_genfiles = False,
     deps = [],
     grpc_plugin = None,
     grpc_plugin_options = [],
     imports = [],
     lang = CPP,
+    paths = [],
     proto_compile = cc_proto_compile,
     protobuf_plugin_options = [],
     protobuf_plugin = None,
@@ -33,6 +34,7 @@ def cc_proto_library(
   args["gen_protobuf_" + lang.name + "_plugin"] = protobuf_plugin
   args["gen_" + lang.name + "_plugin_options"] = protobuf_plugin_options
   args["gen_grpc_" + lang.name + "_plugin"] = grpc_plugin
+  args["paths"] = paths
   args["protoc"] = protoc
   args["protos"] = protos
   args["verbose"] = verbose

bzl/go/rules.bzl

@@ -13,6 +13,7 @@ def go_proto_library(
     grpc_plugin_options = [],
     imports = [],
     lang = GO,
+    paths = [],
     proto_compile = go_proto_compile,
     protobuf_plugin_options = [],
     protobuf_plugin = None,
@@ -33,6 +34,7 @@ def go_proto_library(
   args["gen_protobuf_" + lang.name + "_plugin"] = protobuf_plugin
   args["gen_" + lang.name + "_plugin_options"] = protobuf_plugin_options
   args["gen_grpc_" + lang.name + "_plugin"] = grpc_plugin
+  args["paths"] = paths
   args["protoc"] = protoc
   args["protos"] = protos
   args["verbose"] = verbose

bzl/java/class.bzl

@@ -13,10 +13,10 @@ def _build_generated_files(lang, self):
     protojar = ctx.new_file(srcjar, "%s.jar" % basename)
     self["protojar"] = protojar
     # This will generate the jar inthe source tree itself
-    #self["gendir"] = protojar.short_path
+    #self["outdir"] = protojar.short_path
 
     # This will generate the jar in the BINDIR
-    self["gendir"] = protojar.path
+    self["outdir"] = protojar.path
     self["provides"] += [protojar]
 
 

bzl/protoc.bzl

@@ -78,7 +78,7 @@ def protoc_genrule(name,
 
     invoke("build_generated_filenames", lang, self)
     invoke("build_tools", lang, self)
-    invoke("build_imports", lang, self)
+    invoke("build_paths", lang, self)
     invoke("build_protobuf_invocation", lang, self)
     invoke("build_protobuf_out", lang, self)
     if with_grpc:
@@ -96,20 +96,11 @@ def protoc_genrule(name,
   return struct(**self)
 
 
-def _get_path(ctx, path):
-  if ctx.label.workspace_root:
-    return ctx.label.workspace_root + '/' + path
-  else:
-    return path
-
-
 def _get_gendir(ctx):
   if ctx.attr.output_to_genfiles:
-    outdir = ctx.var["GENDIR"]
+    return ctx.var["GENDIR"]
   else:
-    outdir = ctx.var["BINDIR"]
-
-  return outdir + "/" + ctx.label.package
+    return ctx.var["BINDIR"]
 
 
 def _execute_rule(self):
@@ -118,13 +109,16 @@ def _execute_rule(self):
       fail("Bazel context required for rule execution")
 
   srcfiles = []
-  for src in self["srcs"]:
-    srcfiles += [src.path]
-
+  for src in self["srcfilenames"]:
+    srcfiles += [src]
+    #srcfiles += [src.path]
 
   #self["args"] += ["--descriptor_set_out=%s" % (descriptor_set_file.path)]
 
-  arguments = list(set(self["args"] + ["-I" + i for i in self["imports"]] + srcfiles))
+  arglist = list(set(self["args"]))
+  pathlist = ["--proto_path=" + i for i in set(self["paths"])]
+
+  arguments = arglist + pathlist + srcfiles
   inputs = list(set(self["requires"]))
   outputs = list(set(self["provides"] + ctx.outputs.outs))
 
@@ -166,22 +160,33 @@ def _build_source_files(ctx, self):
         arguments = [srcfile.path, protofile.path],
         command = "cp $1 $2")
       self["srcs"] += [protofile]
-      self["imports"] += [protofile.dirname]
+      self["srcfilenames"] += [srcfile.short_path]
   else:
     if self["verbose"]:
       print("No Copy source files.")
     for srcfile in ctx.files.protos:
       self["srcs"] += [srcfile]
-      self["imports"] += [srcfile.dirname]
+      self["srcfilenames"] += [srcfile.short_path]
+
+  # This is the key to enable imports: protoc can see the entire
+  # source tree from the workspace root.
+  self["paths"] += ["."]
 
 def _protoc_rule_impl(ctx):
 
+  gendir = _get_gendir(ctx)
+  outdir = gendir
+  #outdir = gendir + "/" + ctx.label.package
+
   self = {
     "ctx": ctx,
-    "gendir": _get_gendir(ctx),
+    "gendir": gendir,
+    "outdir": outdir,
     "imports": [],
+    "paths": [],
     "args": [],
     "srcs": [],
+    "srcfilenames": [],
     "requires": [],
     "copy_protos_to_genfiles": getattr(ctx.attr, "copy_protos_to_genfiles", False),
     "provides": [],
@@ -192,11 +197,12 @@ def _protoc_rule_impl(ctx):
 
   # Propogate proto deps: TODO: this is completely untested.
   for dep in ctx.attr.deps:
-    self["imports"] += dep.proto.imports
-    self["requires"] += dep.proto.deps
-    self["srcs"] += dep.proto.srcs
+    if hasattr(dep, "proto"):
+      self["paths"] += dep.proto.paths
+      self["requires"] += dep.proto.deps
+      self["srcs"] += dep.proto.srcs
 
-  # Copy source files over to gendir
+  # Copy source files over to outdir
   _build_source_files(ctx, self)
 
   # Make a list of languages that were specified for this run
@@ -211,6 +217,7 @@ def _protoc_rule_impl(ctx):
 
     invoke("build_generated_files", lang, self)
     invoke("build_imports", lang, self)
+    invoke("build_paths", lang, self)
     invoke("build_protobuf_invocation", lang, self)
     invoke("build_protobuf_out", lang, self)
     if self["with_grpc"]:
@@ -230,6 +237,7 @@ def _protoc_rule_impl(ctx):
     proto=struct(
       srcs = set(self["srcs"]),
       imports = self["imports"],
+      paths = self["paths"],
       deps = self["requires"],
     ),
   )
@@ -254,11 +262,21 @@ def implement(spec):
 
   # ? How to deps interact here?  Don't understand this. Provider
   # aspect is "proto".
-  attrs["deps"] = attr.label_list(providers = ["proto"])
-
-  # Additional include options to protoc.  These should be
-  # directories.  TODO(user): should this be typed as directory only?
-  attrs["imports"] = attr.string_list()
+  #attrs["deps"] = attr.label_list(providers = ["proto"])
+  attrs["deps"] = attr.label_list()
+
+  # Options to be passed to protoc as --proto_path.  Differs from
+  # imports in that these are raw strings rather than labels.
+  attrs["paths"] = attr.string_list()
+
+  # Protos that should be made available for proto imports.  These are
+  # not added as options but rather copied over to the sandbox where
+  # protoc is run, making them available for import. TODO: is this
+  # really needed?  Test it by using the descriptor protos from
+  # google/protobuf.
+  attrs["imports"] = attr.label_list(
+    allow_files = FileType([".proto"]),
+  )
 
   # The list of files the rule generates.  How is this actually being
   # used?
@@ -281,10 +299,9 @@ def implement(spec):
   # Generate the descriptor? Shouldn't we just always generate this?
   #attrs["with_descriptor"] = attr.bool()
 
-  # Implemntation detail that varies between output languages. JAVA:
-  # does not matter.
+  # Implementation detail that varies between output languages.
   attrs["copy_protos_to_genfiles"] = attr.bool(
-    default = True,
+    default = False,
   )
 
   # Flag that sets gen_grpc_{lang} to true for all languages.

examples/helloworld/proto/BUILD

@@ -15,29 +15,34 @@ filegroup(
 # Go proto library implementation cannot accept proto filesets from
 # say, a filegroup.  It needs to be an explicit filename.  This is the
 # reason this rule is in the proto dir.
-go_proto_library(
-    name = "go",
-    #name = "go_default_library", a viable alternative
-    protos = ["helloworld.proto"],
-    verbose = 0,
-    with_grpc = True,
-)
+# go_proto_library(
+#     name = "go",
+#     protos = ["helloworld.proto"],
+#     imports = [
+#         "//examples/proto:srcs",
+#     ],
+#     verbose = 2,
+#     with_grpc = True,
+# )
 
 # cc_proto_library implementation cannot accept proto filesets from
 # say, a filegroup.  It needs to be an explicit filename.  This is the
 # reason this rule is in the proto dir.
 cc_proto_library(
     name = "cpp",
     protos = ["helloworld.proto"],
+    deps = [
+        "//examples/proto:cpplib",
+    ],
     verbose = 0,
     with_grpc = True,
 )
 
-grpc_gateway_proto_library(
-    name = "gw",
-    protos = ["helloworld_gateway.proto"],
-    verbose = 0,
-)
+# grpc_gateway_proto_library(
+#     name = "gw",
+#     protos = ["helloworld_gateway.proto"],
+#     verbose = 0,
+# )
 
 # py_proto_library(
 #     name = "pylib",

examples/helloworld/proto/helloworld.proto

@@ -5,6 +5,8 @@ option java_multiple_files = true;
 option java_package = "org.pubref.rules_protobuf.examples.helloworld";
 option java_outer_classname = "HelloWorldProto";
 
+import "examples/proto/common.proto";
+
 package helloworld;
 
 // The greeting service definition.
@@ -16,6 +18,7 @@ service Greeter {
 // The request message containing the user's name.
 message HelloRequest {
   string name = 1;
+  common.Config config = 2;
 }
 
 // The response message containing the greetings

examples/proto/BUILD

@@ -0,0 +1,22 @@
+package(default_visibility = ["//visibility:public"])
+
+load("//bzl:go/rules.bzl", "go_proto_library")
+load("//bzl:cpp/rules.bzl", "cc_proto_library")
+
+filegroup(
+    name = "srcs",
+    srcs = [
+        "common.proto",
+    ],
+)
+
+go_proto_library(
+    name = "go_default_library",
+    protos = ["common.proto"],
+)
+
+cc_proto_library(
+    name = "cpplib",
+    protos = ["common.proto"],
+    verbose = 2,
+)

examples/proto/README.md

@@ -0,0 +1,3 @@
+This directory contains proto files that are common to different
+examples.  At the moment, there is only one (helloworld).  This
+permits the testing of proto imports.