diff --git a/app/controllers/concerns/rate_limiting.rb b/app/controllers/concerns/rate_limiting.rb index aa8328a2e..61eaea4a7 100644 --- a/app/controllers/concerns/rate_limiting.rb +++ b/app/controllers/concerns/rate_limiting.rb @@ -21,6 +21,7 @@ module RateLimiting # Default rate limit configuration RATE_LIMIT_CONFIG = { recheck: { limit: 3, window: 1.hour }, + retry: { limit: 5, window: 1.hour }, dns_query: { limit: 10, window: 1.minute }, api_call: { limit: 60, window: 1.minute } }.freeze @@ -36,6 +37,17 @@ def rate_limit_recheck ) end + # Rate limit the retry action (SMTP tests are expensive) + # Limits: 5 requests per hour per record per user + def rate_limit_retry + rate_limit( + action: "retry", + scope: [@record.id, current_user.id], + limit: RATE_LIMIT_CONFIG[:retry][:limit], + window: RATE_LIMIT_CONFIG[:retry][:window] + ) + end + # Generic rate limiting method # # @param action [String] The action being rate limited diff --git a/app/controllers/ip_blacklist_records_controller.rb b/app/controllers/ip_blacklist_records_controller.rb index bc27678d2..1d4446b5e 100644 --- a/app/controllers/ip_blacklist_records_controller.rb +++ b/app/controllers/ip_blacklist_records_controller.rb @@ -9,8 +9,9 @@ class IPBlacklistRecordsController < ApplicationController include RateLimiting before_action :admin_required - before_action :load_record, only: [:show, :resolve, :ignore, :recheck] + before_action :load_record, only: [:show, :resolve, :ignore, :recheck, :retry_now] before_action :rate_limit_recheck, only: [:recheck] + before_action :rate_limit_retry, only: [:retry_now] # GET /ip_blacklist_records # List all blacklist records with filtering @@ -159,6 +160,82 @@ def recheck end end + # POST /ip_blacklist_records/:id/retry_now + # Manually trigger an immediate retry test for SMTP-detected blacklists + def retry_now + # Verify this is an SMTP-detected blacklist + unless @record.detected_via_smtp? + message = "Retry is only available for SMTP-detected blacklists" + return respond_to do |format| + format.html { redirect_back fallback_location: ip_blacklist_record_path(@record), alert: message } + format.json { render json: { error: message }, status: :unprocessable_content } + end + end + + # Verify record is still active + unless @record.active? + message = "Cannot retry: blacklist record is already #{@record.status}" + return respond_to do |format| + format.html { redirect_back fallback_location: ip_blacklist_record_path(@record), alert: message } + format.json { render json: { error: message }, status: :unprocessable_content } + end + end + + # Log manual retry trigger + Rails.logger.info "[BLACKLIST RETRY] Manual retry triggered by #{current_user.name} for record #{@record.id} (IP #{@record.ip_address.ipv4}, domain #{@record.destination_domain})" + + # Perform retry + begin + retry_service = IPBlacklist::RetryService.new(@record) + result = retry_service.perform_retry + + case result + when :success + message = "✓ Retry successful! IP #{@record.ip_address.ipv4} is no longer blacklisted for #{@record.destination_domain}. Warmup process started." + notice_type = :notice + when :failed + message = "✗ Retry failed: IP #{@record.ip_address.ipv4} is still blacklisted for #{@record.destination_domain}. Next automatic retry scheduled for #{@record.next_retry_at&.strftime('%Y-%m-%d %H:%M')}. Reason: #{retry_service.error_message}" + notice_type = :alert + when :error + message = "✗ Retry error: #{retry_service.error_message}. Next automatic retry scheduled for #{@record.next_retry_at&.strftime('%Y-%m-%d %H:%M')}." + notice_type = :alert + end + + respond_to do |format| + format.html { redirect_back fallback_location: ip_blacklist_record_path(@record), notice_type => message } + format.json do + render json: { + success: result == :success, + result: result, + message: message, + next_retry_at: @record.next_retry_at, + record: { + id: @record.id, + status: @record.reload.status, + retry_count: @record.retry_count, + retry_result: @record.retry_result, + last_retry_at: @record.last_retry_at + } + } + end + end + rescue StandardError => e + # Log detailed error + error_id = SecureRandom.uuid + Rails.logger.error "[BLACKLIST RETRY] Error ID: #{error_id}" + Rails.logger.error "[BLACKLIST RETRY] Record ID: #{@record.id}, User: #{current_user.name}" + Rails.logger.error "[BLACKLIST RETRY] #{e.class}: #{e.message}" + Rails.logger.error e.backtrace.join("\n") + + error_message = "Retry failed due to an error. Please try again later. (Error ID: #{error_id})" + + respond_to do |format| + format.html { redirect_back fallback_location: ip_blacklist_record_path(@record), alert: error_message } + format.json { render json: { error: error_message, error_id: error_id }, status: :internal_server_error } + end + end + end + private def load_record diff --git a/app/controllers/ip_reputation_controller.rb b/app/controllers/ip_reputation_controller.rb index 139218614..b25e5048d 100644 --- a/app/controllers/ip_reputation_controller.rb +++ b/app/controllers/ip_reputation_controller.rb @@ -93,8 +93,18 @@ def calculate_dashboard_stats def count_healthy_ips total = IPAddress.count - problematic = IPDomainExclusion.select(:ip_address_id).distinct.count + - IPBlacklistRecord.active.select(:ip_address_id).distinct.count + + # Use UNION to get distinct IPs that have either blacklist records OR domain exclusions + # This avoids double-counting IPs that have both + blacklisted_ids = IPBlacklistRecord.active.select(:ip_address_id) + excluded_ids = IPDomainExclusion.select(:ip_address_id) + + problematic = IPAddress + .where(id: blacklisted_ids) + .or(IPAddress.where(id: excluded_ids)) + .distinct + .count + [total - problematic, 0].max end diff --git a/app/lib/ip_blacklist/ip_health_manager.rb b/app/lib/ip_blacklist/ip_health_manager.rb index 642a56206..342c0fc78 100644 --- a/app/lib/ip_blacklist/ip_health_manager.rb +++ b/app/lib/ip_blacklist/ip_health_manager.rb @@ -204,6 +204,9 @@ def handle_smtp_rejection(ip_address, destination_domain, parsed_response, smtp_ Rails.logger.warn "[SMTP REJECTION] Created blacklist record for IP #{ip_address.ipv4} - source: #{parsed_response[:blacklist_source]}" + # Schedule automatic retry for SMTP-detected blacklists (2 days from now) + blacklist_record.schedule_retry! + # Handle the blacklist using existing logic handle_blacklist_detected(blacklist_record) else diff --git a/app/lib/ip_blacklist/notifier.rb b/app/lib/ip_blacklist/notifier.rb index afd821fa7..a0be99e76 100644 --- a/app/lib/ip_blacklist/notifier.rb +++ b/app/lib/ip_blacklist/notifier.rb @@ -118,6 +118,61 @@ def notify_warmup_advanced(ip_address, domain, old_stage, new_stage) send_notifications(event) end + # Notify when retry test succeeds + def notify_retry_success(blacklist_record, test_result) + event = { + event_type: "blacklist_retry_success", + severity: "info", + ip_address: blacklist_record.ip_address.ipv4, + hostname: blacklist_record.ip_address.hostname, + destination_domain: blacklist_record.destination_domain, + blacklist_source: blacklist_record.blacklist_source, + retry_count: blacklist_record.retry_count, + test_result: test_result[:reason], + timestamp: Time.current.iso8601 + } + + send_notifications(event) + end + + # Notify when retry test fails + def notify_retry_failed(blacklist_record, test_result) + event = { + event_type: "blacklist_retry_failed", + severity: "medium", + ip_address: blacklist_record.ip_address.ipv4, + hostname: blacklist_record.ip_address.hostname, + destination_domain: blacklist_record.destination_domain, + blacklist_source: blacklist_record.blacklist_source, + retry_count: blacklist_record.retry_count, + next_retry_at: blacklist_record.next_retry_at&.iso8601, + test_result: test_result[:reason], + smtp_code: test_result[:smtp_code], + smtp_message: test_result[:smtp_message], + timestamp: Time.current.iso8601 + } + + send_notifications(event) + end + + # Notify when retry test encounters an error + def notify_retry_error(blacklist_record, exception) + event = { + event_type: "blacklist_retry_error", + severity: "medium", + ip_address: blacklist_record.ip_address.ipv4, + hostname: blacklist_record.ip_address.hostname, + destination_domain: blacklist_record.destination_domain, + blacklist_source: blacklist_record.blacklist_source, + retry_count: blacklist_record.retry_count, + next_retry_at: blacklist_record.next_retry_at&.iso8601, + error: exception.message, + timestamp: Time.current.iso8601 + } + + send_notifications(event) + end + private def send_notifications(event) @@ -224,6 +279,12 @@ def format_slack_message(event) text = "⚠️ IP #{event[:ip_address]} reputation warning: #{event[:metric_type]} = #{event[:metric_value]} (threshold: #{event[:threshold]})" when "warmup_advanced" text = "📈 IP #{event[:ip_address]} warmup advanced from stage #{event[:old_stage]} to #{event[:new_stage]} for domain #{event[:destination_domain]}" + when "blacklist_retry_success" + text = "✅ IP #{event[:ip_address]} retry successful! No longer blacklisted on #{event[:blacklist_source]} for #{event[:destination_domain]}" + when "blacklist_retry_failed" + text = "❌ IP #{event[:ip_address]} retry failed for #{event[:blacklist_source]} on #{event[:destination_domain]}. Next retry: #{event[:next_retry_at]}" + when "blacklist_retry_error" + text = "⚠️ IP #{event[:ip_address]} retry error for #{event[:blacklist_source]} on #{event[:destination_domain]}: #{event[:error]}" else text = "IP Health Event: #{event[:event_type]}" end diff --git a/app/lib/ip_blacklist/retry_service.rb b/app/lib/ip_blacklist/retry_service.rb new file mode 100644 index 000000000..8b33aec33 --- /dev/null +++ b/app/lib/ip_blacklist/retry_service.rb @@ -0,0 +1,250 @@ +# frozen_string_literal: true + +module IPBlacklist + class RetryService + + attr_reader :blacklist_record, :result, :error_message + + def initialize(blacklist_record) + @blacklist_record = blacklist_record + @result = nil + @error_message = nil + end + + # Perform the retry test by sending a test email + def perform_retry + Rails.logger.info "[BLACKLIST RETRY] Starting retry test for IP #{ip_address.ipv4} on domain #{destination_domain}" + + # Update retry tracking + blacklist_record.update!( + last_retry_at: Time.current, + retry_count: blacklist_record.retry_count + 1 + ) + + begin + # Send test email and analyze result + test_result = send_test_email + + if test_result[:success] + handle_success(test_result) + else + handle_failure(test_result) + end + rescue StandardError => e + handle_error(e) + end + + @result + end + + private + + def ip_address + @ip_address ||= blacklist_record.ip_address + end + + def destination_domain + @destination_domain ||= blacklist_record.destination_domain + end + + def send_test_email + # Find a server that uses this IP address's pool + server = find_test_server + unless server + return { + success: false, + reason: "No server found using IP pool containing #{ip_address.ipv4}" + } + end + + # Generate test recipient address for the destination domain + test_recipient = generate_test_recipient(destination_domain) + + Rails.logger.info "[BLACKLIST RETRY] Sending test email from #{server.permalink} via IP #{ip_address.ipv4} to #{test_recipient}" + + # Create a test message + message = create_test_message(server, test_recipient) + + # Attempt to send using SMTPSender directly with forced IP + attempt_smtp_send(server, message, test_recipient) + end + + def find_test_server + # Find any server that uses this IP's pool + ip_pool = ip_address.ip_pool + return nil unless ip_pool + + # Get the first active server using this pool + Server.where(ip_pool_id: ip_pool.id).first + end + + def generate_test_recipient(domain) + # Generate a test email like: blacklist-test-{timestamp}@{domain} + "blacklist-test-#{Time.current.to_i}@#{domain}" + end + + def create_test_message(server, recipient) + # Create a minimal test message + { + from: "test@#{server.message_db.organization.domains.first&.name || server.permalink}", + to: recipient, + subject: "Postal IP Blacklist Test - #{Time.current.iso8601}", + plain_body: "This is an automated test message from Postal to verify IP reputation.\n\nIP: #{ip_address.ipv4}\nTime: #{Time.current}\n", + message_id: "" + } + end + + def attempt_smtp_send(server, message, recipient) + # Parse recipient domain + domain = recipient.split("@").last + + # Initialize result + result = { success: false, smtp_code: nil, smtp_message: nil } + + begin + # Create a temporary connection to test SMTP + # We'll use Net::SMTP directly to have fine control + require "net/smtp" + + # Get MX records for destination domain + mx_hosts = resolve_mx_records(domain) + if mx_hosts.empty? + return { + success: false, + reason: "No MX records found for #{domain}", + smtp_code: "550", + smtp_message: "Domain has no MX records" + } + end + + # Try first MX host + mx_host = mx_hosts.first + + Rails.logger.info "[BLACKLIST RETRY] Connecting to #{mx_host} for domain #{domain}" + + # Attempt SMTP connection with the specific IP + # Note: Net::SMTP doesn't support binding to specific source IP easily + # We need to use a lower-level approach or rely on routing + + # For now, we'll simulate by checking if we can connect and send MAIL FROM + smtp = Net::SMTP.new(mx_host, 25) + smtp.open_timeout = 30 + smtp.read_timeout = 30 + + # Start SMTP session + smtp.start(server.permalink) do |client| + # Send MAIL FROM + client.mailfrom(message[:from]) + + # Send RCPT TO - this is where blacklist checks usually happen + client.rcptto(recipient) + + # If we get here without exception, the IP is likely not blacklisted + result = { + success: true, + smtp_code: "250", + smtp_message: "Test recipient accepted", + reason: "SMTP server accepted test recipient without rejection" + } + end + rescue Net::SMTPFatalError => e + # 5xx errors - permanent failure (likely blacklisted) + result = { + success: false, + smtp_code: e.message[/\A(\d{3})/, 1], + smtp_message: e.message, + reason: "SMTP permanent error: #{e.message}" + } + rescue Net::SMTPServerBusy => e + # 4xx errors - temporary failure (could be rate limiting) + result = { + success: false, + smtp_code: e.message[/\A(\d{3})/, 1], + smtp_message: e.message, + reason: "SMTP temporary error: #{e.message}" + } + rescue StandardError => e + result = { + success: false, + reason: "Connection error: #{e.class} - #{e.message}" + } + end + + result + end + + def resolve_mx_records(domain) + require "resolv" + resolver = Resolv::DNS.new + mx_records = [] + + begin + resources = resolver.getresources(domain, Resolv::DNS::Resource::IN::MX) + mx_records = resources.sort_by(&:preference).map { |r| r.exchange.to_s } + rescue StandardError => e + Rails.logger.error "[BLACKLIST RETRY] Failed to resolve MX for #{domain}: #{e.message}" + end + + mx_records + end + + def handle_success(test_result) + Rails.logger.info "[BLACKLIST RETRY] ✓ Retry successful for IP #{ip_address.ipv4} on #{destination_domain}" + + # Update blacklist record + blacklist_record.update!( + retry_result: IPBlacklistRecord::RETRY_SUCCESS, + retry_result_details: test_result.to_json, + next_retry_at: nil + ) + + # Mark as resolved and trigger warmup + blacklist_record.mark_resolved! + + @result = :success + @error_message = nil + + # Send notification + notifier = IPBlacklist::Notifier.new + notifier.notify_retry_success(blacklist_record, test_result) + end + + def handle_failure(test_result) + Rails.logger.warn "[BLACKLIST RETRY] ✗ Retry failed for IP #{ip_address.ipv4} on #{destination_domain}: #{test_result[:reason]}" + + # Update blacklist record + blacklist_record.update!( + retry_result: IPBlacklistRecord::RETRY_FAILED, + retry_result_details: test_result.to_json, + next_retry_at: 2.days.from_now # Schedule next retry + ) + + @result = :failed + @error_message = test_result[:reason] + + # Send notification + notifier = IPBlacklist::Notifier.new + notifier.notify_retry_failed(blacklist_record, test_result) + end + + def handle_error(exception) + Rails.logger.error "[BLACKLIST RETRY] ✗ Retry error for IP #{ip_address.ipv4} on #{destination_domain}: #{exception.class} - #{exception.message}" + Rails.logger.error exception.backtrace.join("\n") + + # Update blacklist record + blacklist_record.update!( + retry_result: IPBlacklistRecord::RETRY_ERROR, + retry_result_details: { error: exception.message, backtrace: exception.backtrace[0..5] }.to_json, + next_retry_at: 2.days.from_now # Schedule next retry + ) + + @result = :error + @error_message = exception.message + + # Send notification + notifier = IPBlacklist::Notifier.new + notifier.notify_retry_error(blacklist_record, exception) + end + + end +end diff --git a/app/lib/ip_blacklist/smtp_response_parser.rb b/app/lib/ip_blacklist/smtp_response_parser.rb index 86bae8fea..96f97ae2e 100644 --- a/app/lib/ip_blacklist/smtp_response_parser.rb +++ b/app/lib/ip_blacklist/smtp_response_parser.rb @@ -6,6 +6,7 @@ module IPBlacklist # - Gmail: reputation-based throttling and blocking # - Outlook/Hotmail: IP blocking and reputation issues # - Yahoo: spam filtering and IP blocking + # - iCloud/Apple: policy-based blocking and reputation issues # - Generic DNSBL patterns: Spamhaus, Barracuda, SORBS, SpamCop, etc. # # @example Parse an SMTP error message @@ -136,6 +137,28 @@ class SMTPResponseParser }, ].freeze + # iCloud/Apple-specific patterns (optimized for ReDoS protection) + ICLOUD_PATTERNS = [ + { + regex: /\A.{0,200}554[- ]5\.7\.1.{0,50}\[HM\d+\].{0,50}Message rejected due to local policy/i, + source: "icloud_policy_rejection", + severity: "high", + description: "iCloud policy-based rejection - IP reputation or spam filtering issue" + }, + { + regex: /\A.{0,200}554[- ]5\.7\.1.{0,100}support\.apple\.com.{0,50}HT204137/i, + source: "icloud_policy_rejection", + severity: "high", + description: "iCloud policy-based rejection - IP reputation or spam filtering issue" + }, + { + regex: /\A.{0,200}421[- ]4\.7\.0.{0,50}\[HM\d+\].{0,50}temporarily deferred/i, + source: "icloud_temporary_block", + severity: "medium", + description: "iCloud temporary deferral - possible reputation issue" + }, + ].freeze + # Parse SMTP response message and code # # @param message [String] The SMTP error message @@ -170,8 +193,10 @@ def self.parse(message, smtp_code) begin Timeout.timeout(PARSE_TIMEOUT) do # Check for provider-specific patterns first (most specific) + # iCloud patterns checked before Gmail to avoid conflicts with generic patterns # then generic DNSBL patterns (fallback) - check_gmail_patterns(safe_message, result) || + check_icloud_patterns(safe_message, result) || + check_gmail_patterns(safe_message, result) || check_outlook_patterns(safe_message, result) || check_yahoo_patterns(safe_message, result) || check_generic_dnsbl_patterns(safe_message, result) @@ -267,6 +292,20 @@ def self.check_yahoo_patterns(message, result) false end + # @private + def self.check_icloud_patterns(message, result) + ICLOUD_PATTERNS.each do |pattern| + next unless message =~ pattern[:regex] + + result[:blacklist_detected] = true + result[:blacklist_source] = pattern[:source] + result[:severity] = pattern[:severity] + result[:description] = pattern[:description] + return true + end + false + end + # @private def self.check_generic_dnsbl_patterns(message, result) DNSBL_PATTERNS.each do |pattern, source| diff --git a/app/models/ip_blacklist_record.rb b/app/models/ip_blacklist_record.rb index 7b2a534da..84e24cecc 100644 --- a/app/models/ip_blacklist_record.rb +++ b/app/models/ip_blacklist_record.rb @@ -12,7 +12,12 @@ # detected_at :datetime not null # detection_method :string(255) default("dnsbl_check") # last_checked_at :datetime +# last_retry_at :datetime +# next_retry_at :datetime # resolved_at :datetime +# retry_count :integer default(0), not null +# retry_result :string(255) +# retry_result_details :text(65535) # smtp_response_code :string(255) # smtp_response_message :text(65535) # status :string(255) default("active"), not null @@ -27,8 +32,10 @@ # index_ip_blacklist_records_on_destination_domain (destination_domain) # index_ip_blacklist_records_on_detection_method (detection_method) # index_ip_blacklist_records_on_ip_address_id (ip_address_id) +# index_ip_blacklist_records_on_next_retry_at (next_retry_at) # index_ip_blacklist_records_on_smtp_rejection_event_id (smtp_rejection_event_id) # index_ip_blacklist_records_on_status_and_last_checked_at (status,last_checked_at) +# index_ip_blacklist_records_on_status_and_next_retry_at (status,next_retry_at) # # Foreign Keys # @@ -50,6 +57,13 @@ class IPBlacklistRecord < ApplicationRecord DETECTION_METHODS = [DNSBL_CHECK, SMTP_RESPONSE].freeze + # Retry results + RETRY_SUCCESS = "success" + RETRY_FAILED = "failed" + RETRY_ERROR = "error" + + RETRY_RESULTS = [RETRY_SUCCESS, RETRY_FAILED, RETRY_ERROR].freeze + # Statuses ACTIVE = "active" RESOLVED = "resolved" @@ -75,6 +89,10 @@ class IPBlacklistRecord < ApplicationRecord where(status: ACTIVE) .where("last_checked_at IS NULL OR last_checked_at < ?", 1.hour.ago) } + scope :needs_retry, lambda { + where(status: ACTIVE, detection_method: SMTP_RESPONSE) + .where("next_retry_at IS NOT NULL AND next_retry_at <= ?", Time.current) + } scope :recent, -> { order(detected_at: :desc) } # Instance methods @@ -116,6 +134,22 @@ def detected_via_dnsbl? detection_method == DNSBL_CHECK end + # Schedule next retry (2 days from now) + def schedule_retry! + update!(next_retry_at: 2.days.from_now) + Rails.logger.info "[BLACKLIST RETRY] Scheduled retry for IP #{ip_address.ipv4} on #{destination_domain} at #{next_retry_at}" + end + + # Check if retry is needed + def needs_retry? + active? && detected_via_smtp? && next_retry_at.present? && next_retry_at <= Time.current + end + + # Check if retry can be scheduled + def can_schedule_retry? + active? && detected_via_smtp? + end + private def trigger_recovery_actions diff --git a/app/models/queued_message.rb b/app/models/queued_message.rb index 57cd2a325..c0ba7be13 100644 --- a/app/models/queued_message.rb +++ b/app/models/queued_message.rb @@ -72,6 +72,7 @@ def allocate_ip_address # Reallocate a different IP address for retry attempts (e.g., after a SoftFail). # Tries to select a different IP from the current one if possible. + # Uses domain-aware selection to avoid blacklisted IPs for this destination. def reallocate_ip_address return unless Postal.ip_pools? return if message.nil? @@ -79,11 +80,24 @@ def reallocate_ip_address pool = server.ip_pool_for_message(message) return if pool.nil? + # Extract destination domain from the queued message + destination_domain = domain || extract_domain_from_message + + # Try to get a different IP, preferring domain-aware selection available_ips = pool.ip_addresses.where.not(id: ip_address_id) if available_ips.exists? - new_ip = available_ips.select_by_priority - else + if destination_domain.present? + # Use domain-aware selection that respects blacklists and warmup status + new_ip = available_ips.select_by_priority_for_domain(destination_domain) + else + # Fallback to basic priority selection if domain cannot be determined + new_ip = available_ips.select_by_priority + end + elsif destination_domain.present? # If there's only one IP in the pool, keep the same one + # (but still check domain exclusions for future reference) + new_ip = pool.ip_addresses.select_by_priority_for_domain(destination_domain) + else new_ip = pool.ip_addresses.select_by_priority end diff --git a/app/scheduled_tasks/retry_blacklisted_ips_scheduled_task.rb b/app/scheduled_tasks/retry_blacklisted_ips_scheduled_task.rb new file mode 100644 index 000000000..6d1fe15ea --- /dev/null +++ b/app/scheduled_tasks/retry_blacklisted_ips_scheduled_task.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +class RetryBlacklistedIpsScheduledTask < ApplicationScheduledTask + + def call + logger.info "[BLACKLIST RETRY] Starting automatic retry check for SMTP-detected blacklists" + + records_to_retry = IPBlacklistRecord.needs_retry + retry_count = 0 + success_count = 0 + failure_count = 0 + error_count = 0 + + logger.info "[BLACKLIST RETRY] Found #{records_to_retry.count} blacklist records ready for retry" + + records_to_retry.find_each do |record| + logger.info "[BLACKLIST RETRY] Processing retry for IP #{record.ip_address.ipv4} on domain #{record.destination_domain}" + + begin + retry_service = IPBlacklist::RetryService.new(record) + result = retry_service.perform_retry + + retry_count += 1 + + case result + when :success + success_count += 1 + logger.info "[BLACKLIST RETRY] ✓ IP #{record.ip_address.ipv4} successfully verified for #{record.destination_domain}" + when :failed + failure_count += 1 + logger.warn "[BLACKLIST RETRY] ✗ IP #{record.ip_address.ipv4} still blacklisted for #{record.destination_domain}" + when :error + error_count += 1 + logger.error "[BLACKLIST RETRY] ✗ Error retrying IP #{record.ip_address.ipv4} for #{record.destination_domain}" + end + rescue StandardError => e + error_count += 1 + logger.error "[BLACKLIST RETRY] Exception processing retry for record #{record.id}: #{e.message}" + logger.error e.backtrace.join("\n") + + # Schedule next retry even on exception + begin + record.update(next_retry_at: 2.days.from_now) + rescue StandardError + nil + end + end + end + + logger.info "[BLACKLIST RETRY] Completed. Total: #{retry_count}, Success: #{success_count}, Failed: #{failure_count}, Errors: #{error_count}" + end + + # Run every 6 hours + def self.next_run_after + 6.hours.from_now + end + +end diff --git a/app/senders/smtp_sender.rb b/app/senders/smtp_sender.rb index 8aaaf214d..f77122eac 100644 --- a/app/senders/smtp_sender.rb +++ b/app/senders/smtp_sender.rb @@ -126,12 +126,22 @@ def send_message_to_smtp_client(raw_message, mail_from, rcpt_to, retry_on_connec # Parse SMTP response for blacklist detection (hard bounce) logger.info "About to call handle_smtp_error_response (hard bounce)" - handle_smtp_error_response(e, soft_bounce: false) - logger.info "Finished handle_smtp_error_response (hard bounce)" - - create_result("HardFail", start_time) do |r| - r.details = "Permanent SMTP delivery error when sending to #{@current_endpoint}" - r.output = e.message + blacklist_detected = handle_smtp_error_response(e, soft_bounce: false) + logger.info "Finished handle_smtp_error_response (hard bounce) - blacklist_detected: #{blacklist_detected}" + + # If blacklist detected, convert to SoftFail to allow retry with different IP + if blacklist_detected + logger.warn "Blacklist detected - converting HardFail to SoftFail for retry with different IP" + create_result("SoftFail", start_time) do |r| + r.details = "IP blacklist detected when sending to #{@current_endpoint} - will retry with different IP" + r.output = e.message + r.retry = true + end + else + create_result("HardFail", start_time) do |r| + r.details = "Permanent SMTP delivery error when sending to #{@current_endpoint}" + r.output = e.message + end end rescue StandardError => e logger.error "#{e.class}: #{e.message}" @@ -257,10 +267,11 @@ def logger # # @param exception [Exception] The SMTP exception # @param soft_bounce [Boolean] Whether this is a soft bounce + # @return [Boolean] True if blacklist was detected, false otherwise # def handle_smtp_error_response(exception, soft_bounce:) unless smtp_response_analysis_enabled? - return + return false end # Try to get source IP address @@ -273,18 +284,18 @@ def handle_smtp_error_response(exception, soft_bounce:) source_ip = find_ip_address_by_ip(extracted_ip) unless source_ip logger.debug "[SMTP BLACKLIST] Extracted IP #{extracted_ip} not found in database" - return + return false end else logger.debug "[SMTP BLACKLIST] Could not extract IP from error message" - return + return false end end # Extract SMTP code from exception message smtp_code = extract_smtp_code(exception.message) unless smtp_code - return + return false end # Parse the SMTP response @@ -293,11 +304,15 @@ def handle_smtp_error_response(exception, soft_bounce:) # Handle based on blacklist detection and bounce type if parsed[:blacklist_detected] handle_blacklist_detected_in_smtp(parsed, smtp_code, exception.message, soft_bounce, source_ip) + return true end + + false rescue StandardError => e # Don't let SMTP analysis errors break the main flow logger.error "[SMTP ANALYSIS ERROR] #{e.class}: #{e.message}" logger.error e.backtrace.join("\n") + false end # Handle blacklist detection from SMTP response diff --git a/app/views/ip_blacklist_records/show.html.haml b/app/views/ip_blacklist_records/show.html.haml index 934d8ae00..03e91851f 100644 --- a/app/views/ip_blacklist_records/show.html.haml +++ b/app/views/ip_blacklist_records/show.html.haml @@ -23,6 +23,13 @@ %tr %th Destination Domain %td= @record.destination_domain || "All domains (general blacklisting)" + %tr + %th Detection Method + %td + - if @record.detected_via_smtp? + %span.label.label--blue SMTP Response + - else + %span.label.label--purple DNSBL Check %tr %th Detected At %td @@ -61,6 +68,44 @@ %th Updated At %td= @record.updated_at.strftime("%Y-%m-%d %H:%M:%S") + - if @record.detected_via_smtp? && @record.active? + %h2.sectionTitle Retry Information + + %table.dataTable.u-margin + %tbody + %tr + %th{:width => "30%"} Retry Count + %td= @record.retry_count + - if @record.last_retry_at + %tr + %th Last Retry At + %td + = @record.last_retry_at.strftime("%Y-%m-%d %H:%M:%S") + %br + %small= "(#{time_ago_in_words(@record.last_retry_at)} ago)" + - if @record.retry_result + %tr + %th Last Retry Result + %td + - if @record.retry_result == "success" + %span.label.label--green Success + - elsif @record.retry_result == "failed" + %span.label.label--red Failed + - else + %span.label.label--orange Error + - if @record.next_retry_at + %tr + %th Next Automatic Retry + %td + = @record.next_retry_at.strftime("%Y-%m-%d %H:%M:%S") + %br + %small= "(in #{time_ago_in_words(@record.next_retry_at, include_seconds: false)})" + - else + %tr + %th Next Automatic Retry + %td + %em Not scheduled + - triggered_actions = @record.ip_health_actions.order(created_at: :desc).limit(5) - if triggered_actions.any? %h2.sectionTitle Triggered Actions @@ -99,6 +144,8 @@ .buttonSet.u-margin = link_to "Recheck Delisting", recheck_ip_blacklist_record_path(@record), :method => :post, :class => "button button--primary", :data => {:confirm => "Verify if this IP is still listed?"} + - if @record.detected_via_smtp? + = link_to "Retry Now", retry_now_ip_blacklist_record_path(@record), :method => :post, :class => "button button--info", :data => {:confirm => "Send a test email to verify if the IP is still blacklisted?"} - else %p.u-margin This blacklist record is currently active. You can mark it as resolved if you've @@ -108,6 +155,9 @@ = link_to "Mark as Resolved", resolve_ip_blacklist_record_path(@record), :method => :post, :class => "button button--positive", :data => {:confirm => "Mark this record as resolved? This will start warmup recovery."} = link_to "Mark as Ignored", ignore_ip_blacklist_record_path(@record), :method => :post, :class => "button button--grey", :data => {:confirm => "Mark as false positive? This will prevent automatic actions."} = link_to "Recheck Delisting", recheck_ip_blacklist_record_path(@record), :method => :post, :class => "button", :data => {:confirm => "Verify current blacklist status?"} + - if @record.detected_via_smtp? + = link_to "Retry Now", retry_now_ip_blacklist_record_path(@record), :method => :post, :class => "button button--info", :data => {:confirm => "Send a test email to verify if the IP is still blacklisted?"} + .buttonSet.u-margin = link_to "Back to Records", ip_blacklist_records_path, :class => "button" diff --git a/config/routes.rb b/config/routes.rb index 0cc524e12..47d7d5e42 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -122,6 +122,7 @@ post :resolve post :ignore post :recheck + post :retry_now end end diff --git a/db/migrate/20260130100440_add_retry_fields_to_ip_blacklist_records.rb b/db/migrate/20260130100440_add_retry_fields_to_ip_blacklist_records.rb new file mode 100644 index 000000000..a1fb59dff --- /dev/null +++ b/db/migrate/20260130100440_add_retry_fields_to_ip_blacklist_records.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddRetryFieldsToIPBlacklistRecords < ActiveRecord::Migration[7.1] + + def change + add_column :ip_blacklist_records, :next_retry_at, :datetime + add_column :ip_blacklist_records, :last_retry_at, :datetime + add_column :ip_blacklist_records, :retry_count, :integer, default: 0, null: false + add_column :ip_blacklist_records, :retry_result, :string + add_column :ip_blacklist_records, :retry_result_details, :text + + add_index :ip_blacklist_records, :next_retry_at + add_index :ip_blacklist_records, [:status, :next_retry_at] + end + +end diff --git a/db/schema.rb b/db/schema.rb index 2e414628b..25ef44b33 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2026_01_28_120001) do +ActiveRecord::Schema[7.1].define(version: 2026_01_30_100440) do create_table "additional_route_endpoints", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "route_id" t.string "endpoint_type" @@ -169,12 +169,19 @@ t.string "smtp_response_code" t.text "smtp_response_message" t.integer "smtp_rejection_event_id" + t.datetime "next_retry_at" + t.datetime "last_retry_at" + t.integer "retry_count", default: 0, null: false + t.string "retry_result" + t.text "retry_result_details" t.index ["destination_domain"], name: "index_ip_blacklist_records_on_destination_domain" t.index ["detection_method"], name: "index_ip_blacklist_records_on_detection_method" t.index ["ip_address_id", "destination_domain", "blacklist_source"], name: "index_blacklist_on_ip_domain_source", unique: true t.index ["ip_address_id"], name: "index_ip_blacklist_records_on_ip_address_id" + t.index ["next_retry_at"], name: "index_ip_blacklist_records_on_next_retry_at" t.index ["smtp_rejection_event_id"], name: "index_ip_blacklist_records_on_smtp_rejection_event_id" t.index ["status", "last_checked_at"], name: "index_ip_blacklist_records_on_status_and_last_checked_at" + t.index ["status", "next_retry_at"], name: "index_ip_blacklist_records_on_status_and_next_retry_at" end create_table "ip_domain_exclusions", id: :integer, charset: "utf8mb4", collation: "utf8mb4_uca1400_ai_ci", force: :cascade do |t| diff --git a/spec/factories/ip_blacklist_record_factory.rb b/spec/factories/ip_blacklist_record_factory.rb index 3ba661d57..0b21fe611 100644 --- a/spec/factories/ip_blacklist_record_factory.rb +++ b/spec/factories/ip_blacklist_record_factory.rb @@ -12,7 +12,12 @@ # detected_at :datetime not null # detection_method :string(255) default("dnsbl_check") # last_checked_at :datetime +# last_retry_at :datetime +# next_retry_at :datetime # resolved_at :datetime +# retry_count :integer default(0), not null +# retry_result :string(255) +# retry_result_details :text(65535) # smtp_response_code :string(255) # smtp_response_message :text(65535) # status :string(255) default("active"), not null @@ -27,8 +32,10 @@ # index_ip_blacklist_records_on_destination_domain (destination_domain) # index_ip_blacklist_records_on_detection_method (detection_method) # index_ip_blacklist_records_on_ip_address_id (ip_address_id) +# index_ip_blacklist_records_on_next_retry_at (next_retry_at) # index_ip_blacklist_records_on_smtp_rejection_event_id (smtp_rejection_event_id) # index_ip_blacklist_records_on_status_and_last_checked_at (status,last_checked_at) +# index_ip_blacklist_records_on_status_and_next_retry_at (status,next_retry_at) # # Foreign Keys # diff --git a/spec/lib/ip_blacklist/smtp_response_parser_spec.rb b/spec/lib/ip_blacklist/smtp_response_parser_spec.rb index 3f3d292a1..0f0e6451f 100644 --- a/spec/lib/ip_blacklist/smtp_response_parser_spec.rb +++ b/spec/lib/ip_blacklist/smtp_response_parser_spec.rb @@ -133,7 +133,8 @@ end it "detects Outlook DNSBL block" do - message = "550 SC-001 (BAY004) Unfortunately, messages from [192.0.2.1] weren't sent. Please contact your Internet service provider since part of their network is on our block list (S3140). DNSBL issue." + message = "550 SC-001 (BAY004) Unfortunately, messages from [192.0.2.1] weren't sent. " \ + "Please contact your Internet service provider since part of their network is on our block list (S3140). DNSBL issue." result = described_class.parse(message, "550") expect(result[:blacklist_detected]).to be true @@ -183,6 +184,39 @@ end end + context "iCloud/Apple patterns" do + it "detects iCloud policy rejection with HM code" do + message = "554 5.7.1 [HM08] Message rejected due to local policy. Please visit https://support.apple.com/en-us/HT204137" + result = described_class.parse(message, "554") + + expect(result[:blacklist_detected]).to be true + expect(result[:blacklist_source]).to eq("icloud_policy_rejection") + expect(result[:severity]).to eq("high") + expect(result[:bounce_type]).to eq("hard") + expect(result[:suggested_action]).to eq("pause_immediately") + end + + it "detects iCloud policy rejection with support URL" do + message = "554 5.7.1 Message rejected. See support.apple.com/en-us/HT204137 for more information." + result = described_class.parse(message, "554") + + expect(result[:blacklist_detected]).to be true + expect(result[:blacklist_source]).to eq("icloud_policy_rejection") + expect(result[:severity]).to eq("high") + end + + it "detects iCloud temporary block" do + message = "421 4.7.0 [HM15] Message temporarily deferred. Try again later." + result = described_class.parse(message, "421") + + expect(result[:blacklist_detected]).to be true + expect(result[:blacklist_source]).to eq("icloud_temporary_block") + expect(result[:severity]).to eq("medium") + expect(result[:bounce_type]).to eq("soft") + expect(result[:suggested_action]).to eq("monitor_closely") + end + end + context "Generic DNSBL patterns" do it "detects Spamhaus ZEN listing" do message = "554 Service unavailable; Client host [192.0.2.1] blocked using zen.spamhaus.org" diff --git a/spec/models/ip_blacklist_record_spec.rb b/spec/models/ip_blacklist_record_spec.rb index a4c69f89a..78f05af1c 100644 --- a/spec/models/ip_blacklist_record_spec.rb +++ b/spec/models/ip_blacklist_record_spec.rb @@ -12,7 +12,12 @@ # detected_at :datetime not null # detection_method :string(255) default("dnsbl_check") # last_checked_at :datetime +# last_retry_at :datetime +# next_retry_at :datetime # resolved_at :datetime +# retry_count :integer default(0), not null +# retry_result :string(255) +# retry_result_details :text(65535) # smtp_response_code :string(255) # smtp_response_message :text(65535) # status :string(255) default("active"), not null @@ -27,8 +32,10 @@ # index_ip_blacklist_records_on_destination_domain (destination_domain) # index_ip_blacklist_records_on_detection_method (detection_method) # index_ip_blacklist_records_on_ip_address_id (ip_address_id) +# index_ip_blacklist_records_on_next_retry_at (next_retry_at) # index_ip_blacklist_records_on_smtp_rejection_event_id (smtp_rejection_event_id) # index_ip_blacklist_records_on_status_and_last_checked_at (status,last_checked_at) +# index_ip_blacklist_records_on_status_and_next_retry_at (status,next_retry_at) # # Foreign Keys # @@ -122,4 +129,85 @@ expect(record.ignored?).to be false end end + + describe "retry functionality" do + describe "#schedule_retry!" do + let(:record) { create(:ip_blacklist_record, status: "active", detection_method: "smtp_response") } + + it "sets next_retry_at to 2 days from now" do + Timecop.freeze do + record.schedule_retry! + expect(record.next_retry_at).to be_within(1.second).of(2.days.from_now) + end + end + end + + describe "#needs_retry?" do + it "returns true for active SMTP-detected records with next_retry_at in past" do + record = create(:ip_blacklist_record, + status: "active", + detection_method: "smtp_response", + next_retry_at: 1.hour.ago) + expect(record.needs_retry?).to be true + end + + it "returns false for resolved records" do + record = create(:ip_blacklist_record, + status: "resolved", + detection_method: "smtp_response", + next_retry_at: 1.hour.ago) + expect(record.needs_retry?).to be false + end + + it "returns false for DNSBL-detected records" do + record = create(:ip_blacklist_record, + status: "active", + detection_method: "dnsbl_check", + next_retry_at: 1.hour.ago) + expect(record.needs_retry?).to be false + end + + it "returns false when next_retry_at is in future" do + record = create(:ip_blacklist_record, + status: "active", + detection_method: "smtp_response", + next_retry_at: 1.hour.from_now) + expect(record.needs_retry?).to be false + end + end + + describe ".needs_retry scope" do + let(:ip_address) { create(:ip_address) } + let!(:ready_for_retry) do + create(:ip_blacklist_record, + ip_address: ip_address, + status: "active", + detection_method: "smtp_response", + next_retry_at: 1.hour.ago, + destination_domain: "gmail.com") + end + let!(:not_yet_ready) do + create(:ip_blacklist_record, + ip_address: ip_address, + status: "active", + detection_method: "smtp_response", + next_retry_at: 1.hour.from_now, + destination_domain: "yahoo.com") + end + let!(:resolved_record) do + create(:ip_blacklist_record, + ip_address: ip_address, + status: "resolved", + detection_method: "smtp_response", + next_retry_at: 1.hour.ago, + destination_domain: "outlook.com") + end + + it "returns only records ready for retry" do + expect(IPBlacklistRecord.needs_retry).to include(ready_for_retry) + expect(IPBlacklistRecord.needs_retry).not_to include(not_yet_ready) + expect(IPBlacklistRecord.needs_retry).not_to include(resolved_record) + end + end + end end