A bugfix of when setting null values and integrations tests updated which removes non-allowed the fields param.

Author: mashhurs

lib/logstash/filters/elasticsearch/esql_executor.rb

@@ -91,7 +91,7 @@ def process_response(event, response)
                 # `unless value.nil?` is a part of `drop_null_columns` that if some of the columns' values are not `nil`, `nil` values appear,
                 # we should continuously filter them out to achieve full `drop_null_columns` on each individual row (ideal `LIMIT 1` result)
                 # we also exclude sub-elements of the base field
-                if row && sub_element_mark_map[column] == false
+                if row[index] && sub_element_mark_map[column] == false
                   value_to_set = ESQL_PARSERS_BY_TYPE[column.type].call(row[index])
                   mapped_data[column.name] = value_to_set
                 end

spec/filters/integration/elasticsearch_esql_spec.rb

@@ -72,20 +72,12 @@
       it "processes the event" do
         plugin.filter(event)
         expect(event.get("[@metadata][total_values]")).to eq(expected_count)
-        fields&.each do | old_key, new_key |
-          expect(event.get(new_key)).not_to be(nil)
+        fields&.each do | field |
+          expect(event.get(field)).not_to be(nil)
         end
       end
     end
 
-    describe "LIMIT 1 by default" do
-      let(:config) do
-        super().merge("query" => "FROM #{es_index}")
-      end
-
-      include_examples "ESQL query execution", 1
-    end
-
     describe "with simple FROM query with LIMIT" do
       let(:config) do
         super().merge("query" => "FROM #{es_index} | LIMIT 99")
@@ -102,6 +94,14 @@
       include_examples "ESQL query execution", 2
     end
 
+    describe "with query params" do
+      let(:config) do
+        super().merge("query" => "FROM #{es_index} | WHERE type==?type", "query_params" => { "type" => "b" })
+      end
+
+      include_examples "ESQL query execution", 2
+    end
+
     describe "when invalid query used" do
       let(:config) do
         super().merge("query" => "FROM undefined index | LIMIT 1")
@@ -114,24 +114,27 @@
     end
 
     describe "when field enrichment requested" do
-      fields = {"message" => "target_message", "count" => "target_count"}
       let(:config) do
-        super().merge("query" => "FROM #{es_index} | WHERE type==\"b\" | LIMIT 99", "fields" => fields)
+        super().merge("query" => "FROM #{es_index} | WHERE type==\"b\" | LIMIT 99")
       end
 
-      include_examples "ESQL query execution", 2, fields
+      include_examples "ESQL query execution", 2,  %w[message count]
     end
 
     describe "when non-exist field value appear" do
       let(:config) do
-        super().merge("query" => "FROM #{es_index} | LIMIT 99", "fields" => {"message" => "target_message", "count" => "target_count"})
+        super().merge("query" => "FROM #{es_index}", "target" => "target_field")
       end
 
       it "processes the event" do
         plugin.filter(event)
         expect(event.get("[@metadata][total_values]")).to eq(6)
-        expect(event.get("target_message").size).to eq(6)
-        expect(event.get("target_count").size).to eq(5)
+        expect(event.get("target_field").size).to eq(6)
+        values = event.get("target_field")
+        counts = values.count { |entry| entry.key?("count") }
+        messages = values.count { |entry| entry.key?("message") }
+        expect(counts).to eq(5)
+        expect(messages).to eq(6)
       end
     end
   end