remove obsolete fields and obsolete document_type

Fixes #828

Author: jsvd

lib/logstash/outputs/elasticsearch/common.rb

@@ -78,15 +78,11 @@ def event_action_tuple(event)
         params[:pipeline] = event.sprintf(@pipeline)
       end
 
-      if @parent
-        if @join_field
-          join_value = event.get(@join_field)
-          parent_value = event.sprintf(@parent)
-          event.set(@join_field, { "name" => join_value, "parent" => parent_value })
-          params[routing_field_name] = event.sprintf(@parent)
-        else
-          params[:parent] = event.sprintf(@parent)
-        end
+      if @parent && @join_field
+        join_value = event.get(@join_field)
+        parent_value = event.sprintf(@parent)
+        event.set(@join_field, { "name" => join_value, "parent" => parent_value })
+        params[routing_field_name] = event.sprintf(@parent)
       end
 
       if action == 'update'
@@ -260,16 +256,10 @@ def handle_dlq_status(message, action, status, response)
     DEFAULT_EVENT_TYPE_ES7="_doc".freeze
     def get_event_type(event)
       # Set the 'type' value for the index.
-      type = if @document_type
-               event.sprintf(@document_type)
+      type = if client.maximum_seen_major_version < 7
+               DEFAULT_EVENT_TYPE_ES6
              else
-               if client.maximum_seen_major_version < 6
-                 event.get("type") || DEFAULT_EVENT_TYPE_ES6
-               elsif client.maximum_seen_major_version == 6
-                 DEFAULT_EVENT_TYPE_ES6
-               else
-                 DEFAULT_EVENT_TYPE_ES7
-               end
+               DEFAULT_EVENT_TYPE_ES7
              end
 
       if !(type.is_a?(String) || type.is_a?(Numeric))

lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -17,8 +17,7 @@ def self.included(mod)
       mod.config :index, :validate => :string, :default => DEFAULT_INDEX_NAME
 
       mod.config :document_type, 
-        :validate => :string, 
-        :deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature"
+        :obsolete => "Multiple document types per index were deprecated in Elasticsearch 6.0 and removed entirely in 7.0. We'll now default to 'doc' on ES 6 and '_doc' in 7"
 
       # From Logstash 1.3 onwards, a template is applied to Elasticsearch during
       # Logstash's startup if one with the name `template_name` does not already exist.
@@ -77,7 +76,7 @@ def self.included(mod)
       mod.config :routing, :validate => :string
 
       # For child documents, ID of the associated parent.
-      # This can be dynamic using the `%{foo}` syntax.
+      # This can be dynamic using the `%{foo}` syntax.:w
       mod.config :parent, :validate => :string, :default => nil
 
       # For child documents, name of the join field
@@ -96,10 +95,6 @@ def self.included(mod)
       # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
       mod.config :hosts, :validate => :uri, :default => [::LogStash::Util::SafeURI.new("//127.0.0.1")], :list => true
 
-      mod.config :flush_size, :validate => :number, :obsolete => "This setting is no longer available as we now try to restrict bulk requests to sane sizes. See the 'Batch Sizes' section of the docs. If you think you still need to restrict payloads based on the number, not size, of events, please open a ticket."
-
-      mod.config :idle_flush_time, :validate => :number, :obsolete => "This settings is no longer valid. This was a no-op now as every pipeline batch is flushed synchronously obviating the need for this option."
-
       # Set upsert content for update mode.s
       # Create a new document with this parameter as json string if `document_id` doesn't exists
       mod.config :upsert, :validate => :string, :default => ""

spec/integration/outputs/parent_spec.rb

@@ -151,7 +151,6 @@
             "hosts" => get_host_port,
             "index" => index,
             "parent" => parent_id,
-            "document_type" => type,
             "join_field" => join_field,
             "manage_template" => false
           }
@@ -166,7 +165,6 @@
             "hosts" => get_host_port,
             "index" => index,
             "parent" => "%{link_to}",
-            "document_type" => type,
             "join_field" => join_field,
             "manage_template" => false
           }

spec/unit/outputs/elasticsearch_spec.rb

@@ -43,34 +43,24 @@
     let(:manticore_url) { manticore_urls.first }
 
     describe "getting a document type" do
-      context "if document_type isn't set" do
-        let(:options) { super.merge("document_type" => nil)}
-        context "for 7.x elasticsearch clusters" do
-          let(:maximum_seen_major_version) { 7 }
-          it "should return '_doc'" do
-            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("_doc")
-          end
-        end
-
-        context "for 6.x elasticsearch clusters" do
-          let(:maximum_seen_major_version) { 6 }
-          it "should return 'doc'" do
-            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
-          end
+      context "for 7.x elasticsearch clusters" do
+        let(:maximum_seen_major_version) { 7 }
+        it "should return '_doc'" do
+          expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("_doc")
         end
+      end
 
-        context "for < 6.0 elasticsearch clusters" do
-          let(:maximum_seen_major_version) { 5 }
-          it "should get the type from the event" do
-            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
-          end
+      context "for 6.x elasticsearch clusters" do
+        let(:maximum_seen_major_version) { 6 }
+        it "should return 'doc'" do
+          expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
         end
       end
 
-      context "with 'document type set'" do
-        let(:options) { super.merge("document_type" => "bar")}
-        it "should get the event type from the 'document_type' setting" do
-          expect(subject.send(:get_event_type, LogStash::Event.new())).to eql("bar")
+      context "for < 6.0 elasticsearch clusters" do
+        let(:maximum_seen_major_version) { 5 }
+        it "should return 'doc'" do
+          expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
         end
       end