Fix Post agent to correctly decode server response

Vidas P · Vidas P · commit a5bd807598c9 · 2021-05-10T19:42:43.000+03:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [current]
+### Fixed
+- Post agent will now interpret server responses as UTF-8 with an option
+  to override.
 
 ## [0.9.14.2] - 2021-04-18
 ### Fixed
diff --git a/app/models/agents/post_agent.rb b/app/models/agents/post_agent.rb
@@ -49,6 +49,7 @@ class PostAgent < Agent
           * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
           * `disable_ssl_verification` - Set to `true` to disable ssl verification.
           * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
+          * `force_response_encoding` - Interpret server response in this encoding. Default is UTF-8.
 
         #{receiving_file_handling_agent_description}
 
@@ -114,6 +115,14 @@ def validate_options
         errors.add(:base, 'when content_type is a form, if provided, payload must be a hash')
       end
 
+      if options['force_response_encoding'].present?
+        begin
+          encoding = ::Encoding.find(options['force_response_encoding'])
+        rescue ArgumentError
+          errors.add(:base, "force_response_encoding '#{options['force_response_encoding']}' is invalid")
+        end
+      end
+
       if options.has_key?('emit_messages') && boolify(options['emit_messages']).nil?
         errors.add(:base, 'if provided, emit_messages must be true or false')
       end
@@ -223,8 +232,12 @@ def handle(data, message = Message.new, headers)
 
       if boolify(interpolated['emit_messages'])
         new_message = interpolated['output_mode'].to_s == 'merge' ? message.payload.dup : {}
+
+        encoding = ::Encoding.find(options['force_response_encoding'] || 'UTF-8')
+        body = response.body.force_encoding(encoding).encode('UTF-8')
+
         create_message payload: new_message.merge(
-          body: response.body,
+          body: body,
           headers: normalize_response_headers(response.headers),
           status: response.status
         )
diff --git a/spec/models/agents/post_agent_spec.rb b/spec/models/agents/post_agent_spec.rb
@@ -2,6 +2,7 @@
 require 'ostruct'
 
 describe Agents::PostAgent do
+  let(:response_body) { '<html>an über webpage!</html>' }
   before do
     @valid_options = {
       'post_url' => 'http://www.example.com',
@@ -52,7 +53,7 @@
           raise "unexpected Content-Type: #{content_type}"
         end
       end
-      { status: 200, body: '<html>a webpage!</html>', headers: { 'Content-type' => 'text/html' } }
+      { status: 200, body: response_body, headers: { 'Content-type' => 'text/html' } }
     }
   end
 
@@ -268,7 +269,26 @@
 
         it 'emits the body' do
           @checker.check
-          expect(@checker.messages.last.payload['body']).to eq '<html>a webpage!</html>'
+          expect(@checker.messages.last.payload['body']).to eq '<html>an über webpage!</html>'
+        end
+
+        context 'when response is has no encoding (binary)' do
+          let!(:response_body) { "an \xC3\xBCber webpage!".force_encoding('binary') }
+
+          it 'emits the body decoded as UTF-8' do
+            @checker.check
+            expect(@checker.messages.last.payload['body']).to eq 'an über webpage!'
+          end
+        end
+
+        context 'when response encoding is given explicitly' do
+          let!(:response_body) { "a\xE8i\xFB!".force_encoding('binary') }
+
+          it 'emits the body decoded according to force_response_encoding' do
+            @checker.options['force_response_encoding'] = 'windows-1257'
+            @checker.check
+            expect(@checker.messages.last.payload['body']).to eq 'ačiū!'
+          end
         end
 
         it 'emits the response headers capitalized by default' do
@@ -435,6 +455,14 @@
       expect(@checker).to be_valid
     end
 
+    it 'requires force_response_encoding to be supported encoding' do
+      @checker.options['force_response_encoding'] = 'utf-19'
+      expect(@checker).not_to be_valid
+
+      @checker.options['force_response_encoding'] = 'windows-1257'
+      expect(@checker).to be_valid
+    end
+
     it "requires output_mode to be 'clean' or 'merge', if present" do
       @checker.options['output_mode'] = 'what?'
       expect(@checker).not_to be_valid
