Revert document type deprecation (#844)

This partially reverts commit 1ba256d.

Author: jsvd

lib/logstash/outputs/elasticsearch/common.rb

@@ -77,11 +77,15 @@ def event_action_tuple(event)
         params[:pipeline] = event.sprintf(@pipeline)
       end
 
-      if @parent && @join_field
-        join_value = event.get(@join_field)
-        parent_value = event.sprintf(@parent)
-        event.set(@join_field, { "name" => join_value, "parent" => parent_value })
-        params[routing_field_name] = event.sprintf(@parent)
+      if @parent
+        if @join_field
+          join_value = event.get(@join_field)
+          parent_value = event.sprintf(@parent)
+          event.set(@join_field, { "name" => join_value, "parent" => parent_value })
+          params[routing_field_name] = event.sprintf(@parent)
+        else
+          params[:parent] = event.sprintf(@parent)
+        end
       end
 
       if action == 'update'
@@ -255,10 +259,16 @@ def handle_dlq_status(message, action, status, response)
     DEFAULT_EVENT_TYPE_ES7="_doc".freeze
     def get_event_type(event)
       # Set the 'type' value for the index.
-      type = if client.maximum_seen_major_version < 7
-               DEFAULT_EVENT_TYPE_ES6
+      type = if @document_type
+               event.sprintf(@document_type)
              else
-               DEFAULT_EVENT_TYPE_ES7
+               if client.maximum_seen_major_version < 6
+                 event.get("type") || DEFAULT_EVENT_TYPE_ES6
+               elsif client.maximum_seen_major_version == 6
+                 DEFAULT_EVENT_TYPE_ES6
+               else
+                 DEFAULT_EVENT_TYPE_ES7
+               end
              end
 
       if !(type.is_a?(String) || type.is_a?(Numeric))

lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -18,7 +18,8 @@ def self.included(mod)
       mod.config :index, :validate => :string, :default => DEFAULT_INDEX_NAME
 
       mod.config :document_type, 
-        :obsolete => "Multiple document types per index were deprecated in Elasticsearch 6.0 and removed entirely in 7.0. We'll now default to 'doc' on ES 6 and '_doc' in 7"
+        :validate => :string, 
+        :deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature"
 
       # From Logstash 1.3 onwards, a template is applied to Elasticsearch during
       # Logstash's startup if one with the name `template_name` does not already exist.

spec/integration/outputs/parent_spec.rb

@@ -75,6 +75,7 @@
             "hosts" => get_host_port,
             "index" => index,
             "parent" => parent_id,
+            "document_type" => type,
             "join_field" => join_field,
             "manage_template" => false
           }
@@ -89,6 +90,7 @@
             "hosts" => get_host_port,
             "index" => index,
             "parent" => "%{link_to}",
+            "document_type" => type,
             "join_field" => join_field,
             "manage_template" => false
           }

spec/unit/outputs/elasticsearch_spec.rb

@@ -44,24 +44,34 @@
     let(:manticore_url) { manticore_urls.first }
 
     describe "getting a document type" do
-      context "for 7.x elasticsearch clusters" do
-        let(:maximum_seen_major_version) { 7 }
-        it "should return '_doc'" do
-          expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("_doc")
+      context "if document_type isn't set" do
+        let(:options) { super.merge("document_type" => nil)}
+        context "for 7.x elasticsearch clusters" do
+          let(:maximum_seen_major_version) { 7 }
+          it "should return '_doc'" do
+            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("_doc")
+          end
+        end
+
+        context "for 6.x elasticsearch clusters" do
+          let(:maximum_seen_major_version) { 6 }
+          it "should return 'doc'" do
+            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
+          end
         end
-      end
 
-      context "for 6.x elasticsearch clusters" do
-        let(:maximum_seen_major_version) { 6 }
-        it "should return 'doc'" do
-          expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
+        context "for < 6.0 elasticsearch clusters" do
+          let(:maximum_seen_major_version) { 5 }
+          it "should get the type from the event" do
+            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
+          end
         end
       end
 
-      context "for < 6.0 elasticsearch clusters" do
-        let(:maximum_seen_major_version) { 5 }
-        it "should return 'doc'" do
-          expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
+      context "with 'document type set'" do
+        let(:options) { super.merge("document_type" => "bar")}
+        it "should get the event type from the 'document_type' setting" do
+          expect(subject.send(:get_event_type, LogStash::Event.new())).to eql("bar")
         end
       end
     end