Add 'metadata' option to include @metadata field in output.

Just like the rubydebug codec, being able to include the @metadata
field in the output of the json_lines codec can be useful, for
example for debugging or test tools that prefer machine-readable
output.

Since the Event class's constructor mutates the input hash I had
to refactor some existing tests to send in a clone of the hash
since it'll otherwise end up without a @metadata field.

Author: magnusbaeck

lib/logstash/codecs/json_lines.rb

@@ -28,12 +28,21 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
   # Change the delimiter that separates lines
   config :delimiter, :validate => :string, :default => "\n"
 
+  # If true, the event's metadata (the `@metadata` field) will be
+  # included when used as an output codec.
+  config :metadata, :validate => :boolean, :default => false
+
   public
 
   def register
     @buffer = FileWatch::BufferedTokenizer.new(@delimiter)
     @converter = LogStash::Util::Charset.new(@charset)
     @converter.logger = @logger
+    if @metadata
+      @encoder = method(:encode_with_metadata)
+    else
+      @encoder = method(:encode_default)
+    end
   end
 
   def decode(data, &block)
@@ -43,12 +52,22 @@ def decode(data, &block)
   end
 
   def encode(event)
+    @encoder.call(event)
+  end
+
+  private
+
+  def encode_default(event)
     # Tack on a @delimiter for now because previously most of logstash's JSON
     # outputs emitted one per line, and whitespace is OK in json.
     @on_event.call(event, "#{event.to_json}#{@delimiter}")
   end
 
-  private
+  def encode_with_metadata(event)
+    # Tack on a @delimiter for now because previously most of logstash's JSON
+    # outputs emitted one per line, and whitespace is OK in json.
+    @on_event.call(event, "#{event.to_json_with_metadata}#{@delimiter}")
+  end
 
   # from_json_parse uses the Event#from_json method to deserialize and directly produce events
   def from_json_parse(json, &block)

spec/codecs/json_lines_spec.rb

@@ -120,13 +120,13 @@
   end
 
   context "#encode" do
-    let(:data) { { LogStash::Event::TIMESTAMP => "2015-12-07T11:37:00.000Z", "foo" => "bar", "baz" => {"bah" => ["a","b","c"]}} }
-    let(:event) { LogStash::Event.new(data) }
+    let(:data) { { LogStash::Event::TIMESTAMP => "2015-12-07T11:37:00.000Z", "foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "@metadata" => {"metafield" => "metavalue"} } }
+    let(:event) { LogStash::Event.new(data.clone) }
 
     it "should return json data" do
       got_event = false
       subject.on_event do |e, d|
-        insist { d } == "#{LogStash::Event.new(data).to_json}\n"
+        insist { d } == "#{LogStash::Event.new(data.clone).to_json}\n"
         insist { LogStash::Json.load(d)["foo"] } == data["foo"]
         insist { LogStash::Json.load(d)["baz"] } == data["baz"]
         insist { LogStash::Json.load(d)["bah"] } == data["bah"]
@@ -142,7 +142,19 @@
 
       it "should decode multiple lines separated by the delimiter" do
         subject.on_event do |e, d|
-          insist { d } == "#{LogStash::Event.new(data).to_json}#{delimiter}"
+          insist { d } == "#{LogStash::Event.new(data.clone).to_json}#{delimiter}"
+        end
+        subject.encode(event)
+      end
+    end
+
+    context "when enabling metadata output" do
+      let(:codec_options) { { "metadata" => true } }
+
+      it "should include the @metadata field" do
+        subject.on_event do |e, d|
+          insist { d } == "#{LogStash::Event.new(data.clone).to_json_with_metadata}\n"
+          insist { LogStash::Json.load(d)["@metadata"] } == data["@metadata"]
         end
         subject.encode(event)
       end