From bd88eed12cca2929850056787d26e674382bf753 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 5 Aug 2020 17:41:16 +0900 Subject: [PATCH 1/9] Added lua-resty-http sender --- raven-lua-scm-1.rockspec | 1 + raven/senders/lua-resty-http.lua | 77 ++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 raven/senders/lua-resty-http.lua diff --git a/raven-lua-scm-1.rockspec b/raven-lua-scm-1.rockspec index 6a781ae..0cc0442 100644 --- a/raven-lua-scm-1.rockspec +++ b/raven-lua-scm-1.rockspec @@ -15,6 +15,7 @@ to Sentry.]], dependencies = { "lua >= 5.1", "lua-cjson", + "lua-resty-http" } build = { type = "builtin", diff --git a/raven/senders/lua-resty-http.lua b/raven/senders/lua-resty-http.lua new file mode 100644 index 0000000..a16dfd5 --- /dev/null +++ b/raven/senders/lua-resty-http.lua @@ -0,0 +1,77 @@ +-- vim: st=4 sts=4 sw=4 et: +--- Network backend using [lua-resty-http](https://github.com/ledgetech/lua-resty-http). +--- Supports https, http, and keepalive for better performance. +-- +-- @module raven.senders.lua-resty-http +-- @copyright 2014-2017 CloudFlare, Inc. +-- @license BSD 3-clause (see LICENSE file) + +local util = require 'raven.util' +local http = require 'resty.http' + +local tostring = tostring +local cjson_encode = cjson.encode +local pairs = pairs +local setmetatable = setmetatable +local table_concat = table.concat +local parse_dsn = util.parse_dsn +local generate_auth_header = util.generate_auth_header +local _VERSION = util._VERSION +local _M = {} + +local mt = {} +mt.__index = mt + +function mt:send(json_str) + local httpc = http.new() + local res, err = httpc:request_uri(self.server, { + method = "POST", + headers = { + ['Content-Type'] = 'applicaion/json', + ['User-Agent'] = "raven-lua-http/" .. _VERSION, + ['X-Sentry-Auth'] = generate_auth_header(self), + ["Content-Length"] = tostring(#json_str), + }, + body = cjson_encode(json_str), + keepalive = self.opts.keepalive, + keepalive_timeout = self.opts.keepalive_timeout, + keepalive_pool = self.opts.keepalive_pool + }) + + if not res then + return nil, table_concat(res) + end + + return true +end + +--- Configuration table for the nginx sender. +-- @field dsn DSN string +-- @field verify_ssl Whether or not the SSL certificate is checked (boolean, +-- defaults to false) +-- @field cafile Path to a CA bundle (see the `cafile` parameter in the +-- [newcontext](https://github.com/brunoos/luasec/wiki/LuaSec-0.6#ssl_newcontext) +-- docs) +-- @table sender_conf + +--- Create a new sender object for the given DSN +-- @param conf Configuration table, see @{sender_conf} +-- @return A sender object +function _M.new(conf) + local obj, err = parse_dsn(conf.dsn) + if not obj then + return nil, err + end + + obj.opts = { + verify = conf.verify_ssl or false, + keepalive = conf.keepalive or false, + keepalive_timeout = conf.keepalive_timeout or 0, + keepalive_pool = conf.keepalive_pool or 0 + } + + return setmetatable(obj, mt) +end + +return _M + From f19b54021367d561d08380562c9fd8433fff4e17 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 5 Aug 2020 17:54:59 +0900 Subject: [PATCH 2/9] Fixed Travis CI Issues --- raven/senders/lua-resty-http.lua | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/raven/senders/lua-resty-http.lua b/raven/senders/lua-resty-http.lua index a16dfd5..ea5e136 100644 --- a/raven/senders/lua-resty-http.lua +++ b/raven/senders/lua-resty-http.lua @@ -10,8 +10,7 @@ local util = require 'raven.util' local http = require 'resty.http' local tostring = tostring -local cjson_encode = cjson.encode -local pairs = pairs +local csjon = cjson local setmetatable = setmetatable local table_concat = table.concat local parse_dsn = util.parse_dsn @@ -32,14 +31,14 @@ function mt:send(json_str) ['X-Sentry-Auth'] = generate_auth_header(self), ["Content-Length"] = tostring(#json_str), }, - body = cjson_encode(json_str), + body = cjson.encode(json_str), keepalive = self.opts.keepalive, keepalive_timeout = self.opts.keepalive_timeout, keepalive_pool = self.opts.keepalive_pool }) if not res then - return nil, table_concat(res) + return nil, table_concat(err) end return true From 72aafcc06f7f04c52300e568d7824834e9f63e31 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 5 Aug 2020 17:57:45 +0900 Subject: [PATCH 3/9] Fixed typo --- raven/senders/lua-resty-http.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raven/senders/lua-resty-http.lua b/raven/senders/lua-resty-http.lua index ea5e136..b3ffbca 100644 --- a/raven/senders/lua-resty-http.lua +++ b/raven/senders/lua-resty-http.lua @@ -10,7 +10,7 @@ local util = require 'raven.util' local http = require 'resty.http' local tostring = tostring -local csjon = cjson +local cjson = cjson local setmetatable = setmetatable local table_concat = table.concat local parse_dsn = util.parse_dsn From 36ef53ba784ae9fe1b2fd755b2028765f6412f67 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 5 Aug 2020 18:02:03 +0900 Subject: [PATCH 4/9] Added function description --- raven/senders/lua-resty-http.lua | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/raven/senders/lua-resty-http.lua b/raven/senders/lua-resty-http.lua index b3ffbca..7343fbc 100644 --- a/raven/senders/lua-resty-http.lua +++ b/raven/senders/lua-resty-http.lua @@ -44,13 +44,16 @@ function mt:send(json_str) return true end ---- Configuration table for the nginx sender. +--- Configuration table for the lua-resty-http. -- @field dsn DSN string -- @field verify_ssl Whether or not the SSL certificate is checked (boolean, -- defaults to false) --- @field cafile Path to a CA bundle (see the `cafile` parameter in the --- [newcontext](https://github.com/brunoos/luasec/wiki/LuaSec-0.6#ssl_newcontext) --- docs) +-- @field keepalive Whether or not to keep connection alive (boolean, +-- defaults to false) +-- @field keepalive_timeout The maximum number of connections in the keepalive pool(int, +-- defaults to 0) +-- @field keepalive_pool Whether or not to keep connection alive (int, +-- defaults to 0) -- @table sender_conf --- Create a new sender object for the given DSN From 2c7df897cb097656d9a7e575304954d537d19a64 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 5 Aug 2020 18:17:19 +0900 Subject: [PATCH 5/9] Fixed Cjson Issue I need some sleep --- raven/senders/lua-resty-http.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raven/senders/lua-resty-http.lua b/raven/senders/lua-resty-http.lua index 7343fbc..3cde1eb 100644 --- a/raven/senders/lua-resty-http.lua +++ b/raven/senders/lua-resty-http.lua @@ -8,9 +8,9 @@ local util = require 'raven.util' local http = require 'resty.http' +local cjson = require 'cjson' local tostring = tostring -local cjson = cjson local setmetatable = setmetatable local table_concat = table.concat local parse_dsn = util.parse_dsn From d068e38bd29523edfec2b0f0777360c32897a551 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Sat, 8 Aug 2020 16:12:55 +0900 Subject: [PATCH 6/9] Requested changes - Removed lua-resty-htt pfrom rockspec - Removed table_concat when returning error from request - Added if statement to catch non 200 status code response - Added ngx.timer support --- raven-lua-scm-1.rockspec | 3 +- raven/senders/lua-resty-http.lua | 119 +++++++++++++++++++++++++++---- 2 files changed, 106 insertions(+), 16 deletions(-) diff --git a/raven-lua-scm-1.rockspec b/raven-lua-scm-1.rockspec index 0cc0442..bcda39e 100644 --- a/raven-lua-scm-1.rockspec +++ b/raven-lua-scm-1.rockspec @@ -14,8 +14,7 @@ to Sentry.]], } dependencies = { "lua >= 5.1", - "lua-cjson", - "lua-resty-http" + "lua-cjson" } build = { type = "builtin", diff --git a/raven/senders/lua-resty-http.lua b/raven/senders/lua-resty-http.lua index 3cde1eb..1da93e0 100644 --- a/raven/senders/lua-resty-http.lua +++ b/raven/senders/lua-resty-http.lua @@ -8,11 +8,16 @@ local util = require 'raven.util' local http = require 'resty.http' -local cjson = require 'cjson' +local log = ngx.log +local ERR = ngx.ERR +local CRIT = ngx.CRIT +local ngx_timer_at = ngx.timer.at +local ngx_socket = ngx.socket +local ngx_get_phase = ngx.get_phase local tostring = tostring local setmetatable = setmetatable -local table_concat = table.concat +local table_remove = table.remove local parse_dsn = util.parse_dsn local generate_auth_header = util.generate_auth_header local _VERSION = util._VERSION @@ -21,27 +26,107 @@ local _M = {} local mt = {} mt.__index = mt -function mt:send(json_str) +local function send_msg(self, msg) local httpc = http.new() - local res, err = httpc:request_uri(self.server, { - method = "POST", + local res, err = httpc:request_uri(self.server, msg) + + if not res then + return nil, err + end + + if res.status ~= 200 then + return nil, res.body or 'Sentry responded with status code ' .. res.status + end + + return true +end + +local function consume_queue(premature, self) + if premature then + return + end + + local ok, err = xpcall(function() + local queue = self.queue + + while #queue > 0 do + local msg = queue[1] + + local ok, err = send_msg(self, msg) + if not ok then + log(ERR, 'Raven failed to send message: ', err) + end + + table_remove(queue, 1) + end + end, debug.traceback) + + if not ok then + log(ERR, 'Raven failed to run the async sender task: ', err) + end + + self.task_running = false +end + +local function process_queue(self) + if not self.task_running then + local ok, err = ngx_timer_at(0, consume_queue, self) + if not ok then + return nil, 'Failed to create timer: ' .. err + end + + self.task_running = true + end + + return true +end + +function mt:send(json_str) + -- Prepare http request config + local msg = { + method = 'POST', headers = { ['Content-Type'] = 'applicaion/json', - ['User-Agent'] = "raven-lua-http/" .. _VERSION, + ['User-Agent'] = 'raven-lua-http/' .. _VERSION, ['X-Sentry-Auth'] = generate_auth_header(self), - ["Content-Length"] = tostring(#json_str), + ['Content-Length'] = tostring(#json_str), }, - body = cjson.encode(json_str), + body = json_str, + ssl_verify = self.opts.verify_ssl, keepalive = self.opts.keepalive, keepalive_timeout = self.opts.keepalive_timeout, keepalive_pool = self.opts.keepalive_pool - }) + } - if not res then - return nil, table_concat(err) - end + -- Cosocket is only available in certain phases + local phase = ngx_get_phase() + if phase == 'rewrite' or phase == 'access' or phase == 'content' or phase == 'timer' or phase == 'ssl_cert' or phase == 'ssl_session_fetch' then + return send_msg(self, msg) + else + local queue = self.queue + local queue_size = #queue - return true + if queue_size <= self.queue_limit then + -- Add message to queue + queue[queue_size + 1] = msg + + -- Process queue + local ok, err = process_queue(self) + if not ok then + return nil, err + end + else + -- Queue is full, process queue if not running + local ok, err = process_queue(self) + if not ok then + return nil, err + end + + return nil, 'Queue is full, dropping message' + end + + return true + end end --- Configuration table for the lua-resty-http. @@ -54,6 +139,8 @@ end -- defaults to 0) -- @field keepalive_pool Whether or not to keep connection alive (int, -- defaults to 0) +-- @field queue_limit Maximum number of events in the queue (int, +-- defaults to 10) -- @table sender_conf --- Create a new sender object for the given DSN @@ -66,12 +153,16 @@ function _M.new(conf) end obj.opts = { - verify = conf.verify_ssl or false, + verify_ssl = conf.verify_ssl or false, keepalive = conf.keepalive or false, keepalive_timeout = conf.keepalive_timeout or 0, keepalive_pool = conf.keepalive_pool or 0 } + obj.queue = {} + obj.queue_limit = conf.queue_limit or 10 + obj.task_running = false + return setmetatable(obj, mt) end From 7a5146bf3860f7d1833092cf94226df0570ce063 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Sat, 8 Aug 2020 16:20:39 +0900 Subject: [PATCH 7/9] Fixed Travis CI Errors --- raven/senders/lua-resty-http.lua | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/raven/senders/lua-resty-http.lua b/raven/senders/lua-resty-http.lua index 1da93e0..0ac6e82 100644 --- a/raven/senders/lua-resty-http.lua +++ b/raven/senders/lua-resty-http.lua @@ -11,9 +11,7 @@ local http = require 'resty.http' local log = ngx.log local ERR = ngx.ERR -local CRIT = ngx.CRIT local ngx_timer_at = ngx.timer.at -local ngx_socket = ngx.socket local ngx_get_phase = ngx.get_phase local tostring = tostring local setmetatable = setmetatable @@ -69,7 +67,7 @@ local function consume_queue(premature, self) end local function process_queue(self) - if not self.task_running then + if not self.task_running then local ok, err = ngx_timer_at(0, consume_queue, self) if not ok then return nil, 'Failed to create timer: ' .. err @@ -100,7 +98,13 @@ function mt:send(json_str) -- Cosocket is only available in certain phases local phase = ngx_get_phase() - if phase == 'rewrite' or phase == 'access' or phase == 'content' or phase == 'timer' or phase == 'ssl_cert' or phase == 'ssl_session_fetch' then + if (phase == 'rewrite' or + phase == 'access' or + phase == 'content' or + phase == 'timer' or + phase == 'ssl_cert' or + phase == 'ssl_session_fetch') + then return send_msg(self, msg) else local queue = self.queue From 2c7f8a6d1cf53d76a7022d8d0d4699bd9ad4f43a Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Sat, 8 Aug 2020 16:35:54 +0900 Subject: [PATCH 8/9] Update lua-resty-http.lua Changed ngx.log to util.errlog function --- raven/senders/lua-resty-http.lua | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/raven/senders/lua-resty-http.lua b/raven/senders/lua-resty-http.lua index 0ac6e82..ccbe178 100644 --- a/raven/senders/lua-resty-http.lua +++ b/raven/senders/lua-resty-http.lua @@ -9,8 +9,6 @@ local util = require 'raven.util' local http = require 'resty.http' -local log = ngx.log -local ERR = ngx.ERR local ngx_timer_at = ngx.timer.at local ngx_get_phase = ngx.get_phase local tostring = tostring @@ -52,7 +50,7 @@ local function consume_queue(premature, self) local ok, err = send_msg(self, msg) if not ok then - log(ERR, 'Raven failed to send message: ', err) + return util.errlog('Raven failed to send message: ', err) end table_remove(queue, 1) @@ -60,7 +58,7 @@ local function consume_queue(premature, self) end, debug.traceback) if not ok then - log(ERR, 'Raven failed to run the async sender task: ', err) + util.errlog('Raven failed to run the async sender task: ', err) end self.task_running = false From 94c0225e1a461762c1a14238a34d11f1fada5a98 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Sat, 8 Aug 2020 17:02:32 +0900 Subject: [PATCH 9/9] Added async config option --- raven/senders/lua-resty-http.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/raven/senders/lua-resty-http.lua b/raven/senders/lua-resty-http.lua index ccbe178..89dd271 100644 --- a/raven/senders/lua-resty-http.lua +++ b/raven/senders/lua-resty-http.lua @@ -50,6 +50,7 @@ local function consume_queue(premature, self) local ok, err = send_msg(self, msg) if not ok then + -- Would we want to return here to preserve message in queue or remove it losing logs? return util.errlog('Raven failed to send message: ', err) end @@ -96,7 +97,8 @@ function mt:send(json_str) -- Cosocket is only available in certain phases local phase = ngx_get_phase() - if (phase == 'rewrite' or + if (self.async) and ( + phase == 'rewrite' or phase == 'access' or phase == 'content' or phase == 'timer' or @@ -143,6 +145,9 @@ end -- defaults to 0) -- @field queue_limit Maximum number of events in the queue (int, -- defaults to 10) +-- @field async Always send message asynchronously, even when it can be sent +-- right away. This is to prevent to slow down processing while contacting the +-- Sentry server. (default: false) -- @table sender_conf --- Create a new sender object for the given DSN @@ -164,6 +169,7 @@ function _M.new(conf) obj.queue = {} obj.queue_limit = conf.queue_limit or 10 obj.task_running = false + obj.async = conf.async or false return setmetatable(obj, mt) end