Add more associations and improve performance and warnings

adminapp/src/shared/react/useDebugEffect.jsx

@@ -0,0 +1,22 @@
+import React from "react";
+
+/**
+ * Just a `React.useEffect(cb, [])` that does not fire unless in development mode.
+ * @param cb
+ * @param {Array=} deps Dependency list. If not given, use empty list.
+ * @param {boolean=} once If true, fire just once, even in strict mode.
+ */
+export default function useDebugEffect(cb, { deps, once } = {}) {
+  const calledRef = React.useRef(false);
+  React.useEffect(() => {
+    if (process.env.NODE_ENV !== "development") {
+      return;
+    }
+    if (once && calledRef.current) {
+      return;
+    }
+    cb();
+    calledRef.current = true;
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, deps || []);
+}

db/migrations/111_combined_notes.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    create_view :support_notes_combined_view,
+                from(:support_notes).
+                  left_join(:support_notes_members, {note_id: :id}).
+                  select(
+                    Sequel[:support_notes].*,
+                    :member_id,
+                    Sequel[nil].as(:verification_id),
+                  ).
+                  union(
+                    from(:support_notes).
+                      left_join(:support_notes_organization_membership_verifications, {note_id: :id}).
+                      left_join(:organization_membership_verifications, {id: :verification_id}).
+                      left_join(:organization_memberships, {id: :membership_id}).
+                      select(
+                        Sequel[:support_notes].*,
+                        :member_id,
+                        :verification_id,
+                      ),
+                  ).order(Sequel.desc(Sequel.function(:coalesce, :edited_at, :created_at)), :id)
+  end
+  down do
+    drop_view :support_notes_combined_view
+  end
+end

lib/sequel/plugins/efficient_each.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Sequel::Plugins::EfficientEach
+  class UnknownAssociation < ArgumentError; end
+
+  DEFAULT_OPTIONS = {
+    page_size: 100,
+  }.freeze
+
+  class << self
+    def configure(model, opts=DEFAULT_OPTIONS)
+      opts = DEFAULT_OPTIONS.merge(opts)
+      model.efficient_each_page_size = opts[:page_size]
+    end
+  end
+
+  module ClassMethods
+    attr_accessor :efficient_each_page_size
+
+    def inherited(subclass)
+      super
+      [:efficient_each_page_size].each do |m|
+        subclass.send("#{m}=", self.send(m))
+      end
+    end
+  end
+
+  module DatasetMethods
+    # Call a block for each row in a dataset.
+    # This is the same as paged_each or use_cursor.each, except that for each page,
+    # rows are re-fetched using self.where(primary_key => [pks]).all to enable eager loading.
+    #
+    # @param page_size [Integer] Size of each page. Smaller uses less memory.
+    # @param order [Symbol] Column to order by. Default to primary key.
+    # @param yield_page [true,false] If true, yield the page to the block, rather than individual rows.
+    #   Helpful when bulk processing.
+    #
+    # (Note that paged_each does not do eager loading, which makes enumerating model associations very slow)
+    def each_cursor_page(page_size: nil, order: nil, yield_page: false, &block)
+      raise LocalJumpError unless block
+      raise "dataset requires a use_cursor method, class may need `extension(:pagination)`" unless
+        self.respond_to?(:use_cursor)
+      model = self.model
+      page_size ||= model.efficient_each_page_size
+      pk = model.primary_key
+      order ||= pk
+      current_chunk_pks = []
+      order = [order] unless order.respond_to?(:to_ary)
+      self.naked.select(pk).order(*order).use_cursor(rows_per_fetch: page_size, hold: true).each do |row|
+        current_chunk_pks << row[pk]
+        next if current_chunk_pks.length < page_size
+        page = model.where(pk => current_chunk_pks).order(*order).all
+        current_chunk_pks.clear
+        yield_page ? yield(page) : page.each(&block)
+      end
+      remainder = model.where(pk => current_chunk_pks).order(*order).all
+      yield_page && !remainder.empty? ? yield(remainder) : remainder.each(&block)
+    end
+  end
+
+  module InstanceMethods
+    def efficient_each(association_name, &)
+      return enum_for(:efficient_each, association_name) unless block_given?
+
+      assoc = self.class.association_reflection(association_name)
+      raise UnknownAssociation, "#{self.class.name} has no association :#{association_name}" if
+        assoc.nil?
+      loaded = self.associations[association_name]
+      unless loaded.nil?
+        loaded.each(&)
+        return nil
+      end
+      dataset = self.send(assoc.fetch(:dataset_method))
+      pagecount = 0
+      prev_page = []
+      dataset.each_cursor_page(yield_page: true) do |page|
+        pagecount += 1
+        prev_page = page
+        page.each(&)
+      end
+      self.associations[association_name] = prev_page if pagecount < 2
+      return nil
+    end
+  end
+end

lib/sequel/plugins/large_association_warning.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Sequel::Plugins::LargeAssociationWarning
+  DEFAULT_CALLBACK = lambda do |m, assoc, array|
+    Appydays::Loggable[m].warn(
+      "large_assocation_loaded",
+      model_pk: m.primary_key,
+      model_type: m.class.name,
+      model_association: assoc,
+      model_association_size: array.size,
+    )
+  end
+
+  DEFAULT_OPTIONS = {
+    threshold: 100,
+    callback: DEFAULT_CALLBACK,
+  }.freeze
+
+  class << self
+    attr_reader :warned_associations
+
+    def configure(model, opts=DEFAULT_OPTIONS)
+      opts = DEFAULT_OPTIONS.merge(opts)
+      model.large_association_warning_threshold = opts[:threshold]
+      model.large_association_warning_callback = opts[:callback]
+      @warned_associations = Set.new
+    end
+  end
+
+  module ClassMethods
+    attr_accessor :large_association_warning_threshold, :large_association_warning_callback
+
+    def inherited(subclass)
+      super
+      [:large_association_warning_threshold, :large_association_warning_callback].each do |m|
+        subclass.send("#{m}=", self.send(m))
+      end
+    end
+  end
+
+  module InstanceMethods
+    def load_associated_objects(opts, dynamic_opts={})
+      results = super
+      if results.is_a?(Array) && results.size > model.large_association_warning_threshold
+        assoc = opts.fetch(:name)
+        warn_key = [self.class, assoc]
+        unless Sequel::Plugins::LargeAssociationWarning.warned_associations.include?(warn_key)
+          Sequel::Plugins::LargeAssociationWarning.warned_associations.add(warn_key)
+          model.large_association_warning_callback[self, assoc, results]
+        end
+      end
+      return results
+    end
+  end
+end

lib/suma/analytics/model.rb

@@ -29,8 +29,10 @@ class RowMismatch < StandardError; end
         max_connections: self.max_connections,
         pool_timeout: self.pool_timeout,
       }
-      db = Sequel.connect(self.uri, options)
-      self.db = db
+      if self.guard_db_reconnect?(self.uri, options)
+        db = Sequel.connect(self.uri, options)
+        self.db = db
+      end
     end
   end
 

lib/suma/member.rb

@@ -125,6 +125,18 @@ def initialize(reason)
                join_table: :support_notes_members,
                left_key: :member_id,
                order: order_desc
+  many_to_many :combined_notes,
+               class: "Suma::Support::Note",
+               eager_loader: (lambda do |eo|
+                 eo[:rows].each { |p| p.associations[:combined_notes] = [] }
+                 ds = self.db[:support_notes_combined_view].where(member_id: eo[:id_map].keys)
+                 ds.all.each do |note|
+                   member = eo[:id_map][note[:member_id]].first
+                   member.associations[:combined_notes] << note
+                 end
+               end) do |_ds|
+    Suma::Support::Note.for_member(self)
+  end
 
   one_to_many :eligibility_assignments, class: "Suma::Eligibility::Assignment", order: order_desc
   one_to_many :expanded_eligibility_assignments,
@@ -276,15 +288,6 @@ def default_payment_instrument
     return self.public_payment_instruments.find { |pi| pi.status == :ok }
   end
 
-  def combined_notes
-    ds = Suma::Support::Note.combine_datasets(
-      Sequel[members: self],
-      Sequel[organization_membership_verifications: Suma::Organization::Membership::Verification.
-        where(membership: self.organization_memberships_dataset)],
-    )
-    return ds.all
-  end
-
   # @return [Suma::Member::StripeAttributes]
   def stripe
     return @stripe ||= Suma::Member::StripeAttributes.new(self)

lib/suma/organization/membership/verification.rb

@@ -44,6 +44,31 @@ class Suma::Organization::Membership::Verification < Suma::Postgres::Model(:orga
                join_table: :support_notes_organization_membership_verifications,
                left_key: :verification_id,
                order: order_desc
+  many_to_many :combined_notes,
+               class: "Suma::Support::Note",
+               eager_loader: (lambda do |eo|
+                 eo[:rows].each { |p| p.associations[:combined_notes] = [] }
+                 verifications_for_member_ids = {}
+                 verifications_by_ids = {}
+                 eo[:rows].each do |v|
+                   verifications_for_member_ids[v.membership.member.id] ||= []
+                   verifications_for_member_ids[v.membership.member.id] << v
+                   verifications_by_ids[v.id] = v
+                 end
+                 ds = self.db[:support_notes_combined_view].where(Sequel[member_id: verifications_for_member_ids.keys])
+                 ds.all.each do |note|
+                   if (source_id = note[:verification_id])
+                     verifications_by_ids[source_id].associations[:combined_notes] << note
+                   else
+                     verifications_for_member_ids[note[:member_id]].each do |v|
+                       v.associations[:combined_notes] << note
+                     end
+                   end
+                 end
+               end) do |_ds|
+    Suma::Support::Note.for_verification(self)
+  end
+
   many_to_one :owner, class: "Suma::Member"
 
   many_to_one :front_partner_conversation,
@@ -340,8 +365,6 @@ def find_duplicates = DuplicateFinder.lookup_matches(self)
   # Duplicates are stored sorted so we can use the 0th item.
   def duplicate_risk = self.find_duplicates.first&.max_risk
 
-  def combined_notes = Suma::Support::Note.combine_instances(self.notes, self.membership.member.notes)
-
   def rel_admin_link = "/membership-verification/#{self.id}"
 
   def hybrid_search_fields

lib/suma/payment/card.rb

@@ -17,6 +17,15 @@ class Suma::Payment::Card < Suma::Postgres::Model(:payment_cards)
   one_to_many :originated_funding_stripe_card_strategies,
               key: :originating_card_id,
               class: "Suma::Payment::FundingTransaction::StripeCardStrategy"
+  many_through_many :originated_funding_transactions,
+                    [
+                      [:payment_funding_transaction_stripe_card_strategies, :originating_card_id, :id],
+                    ],
+                    class: "Suma::Payment::FundingTransaction",
+                    left_primary_key: :id,
+                    right_primary_key: :stripe_card_strategy_id,
+                    read_only: true,
+                    order: [:created_at, :id]
 
   dataset_module do
     def usable_for_funding = self.unexpired_as_of(Time.now)
@@ -63,9 +72,6 @@ def refetch_remote_data
     @stripe_data = nil
   end
 
-  # Could move this to an association later
-  def originated_funding_transactions = self.originated_funding_stripe_card_strategies.map(&:funding_transaction)
-
   def _external_links_self
     return [
       self._external_link(

lib/suma/payment/ledger.rb

@@ -230,7 +230,7 @@ def category_used_to_purchase(has_vnd_svc_categories)
   def find_unbalanced_counterparty_ledgers(include_all: false)
     platform_account = Suma::Payment::Account.lookup_platform_account
     totals_by_ledger = {}
-    self.combined_book_transactions.each do |bx|
+    self.efficient_each(:combined_book_transactions).each do |bx|
       if bx.originating_ledger === self
         counterparty = bx.receiving_ledger
         amount = bx.amount * -1

lib/suma/postgres/model.rb

@@ -35,6 +35,8 @@ class Suma::Postgres::Model
 
     setting :extension_schema, "public"
 
+    setting :large_association_warning_threshold, 500
+
     # The number of (Float) seconds that should be considered "slow" for a
     # single query; queries that take longer than this amount of time will be logged
     # at `warn` level.
@@ -58,16 +60,36 @@ class Suma::Postgres::Model
         pool_timeout: self.pool_timeout,
         log_warn_duration: self.slow_query_seconds,
       }
-      db = Sequel.connect(self.uri, options)
-      db.extension(:pagination)
-      db.extension(:pg_json)
-      db.extension(:pg_inet)
-      db.extension(:pg_array)
-      db.extension(:pg_streaming)
-      db.extension(:pg_range)
-      db.extension(:pg_interval)
-      db.extension(:pretty_table)
-      self.db = db
+      if self.guard_db_reconnect?(self.uri, options)
+        db = Sequel.connect(self.uri, options)
+        db.extension(:pagination)
+        db.extension(:pg_json)
+        db.extension(:pg_inet)
+        db.extension(:pg_array)
+        db.extension(:pg_streaming)
+        db.extension(:pg_range)
+        db.extension(:pg_interval)
+        db.extension(:pretty_table)
+        self.db = db
+      end
+
+      plugin :large_association_warning,
+             threshold: self.large_association_warning_threshold,
+             callback: lambda { |m, assoc, array|
+               Sentry.capture_message("Large association loaded") do |scope|
+                 scope.set_extras(
+                   model_pk: m.pk,
+                   model_type: m.class.name,
+                   model_association: assoc,
+                   model_association_size: array.size,
+                 )
+               end
+               Sequel::Plugins::LargeAssociationWarning::DEFAULT_CALLBACK[m, assoc, array]
+             }
+      plugin :efficient_each,
+             # Use a page size of the large warning threshold,
+             # as this is what we consider a reasonable page size.
+             page_size: self.large_association_warning_threshold
     end
   end