diff --git a/app/controllers/concerns/dashboard_data.rb b/app/controllers/concerns/dashboard_data.rb deleted file mode 100644 index d720393d5..000000000 --- a/app/controllers/concerns/dashboard_data.rb +++ /dev/null @@ -1,519 +0,0 @@ -module DashboardData - extend ActiveSupport::Concern - - FILTER_OPTIONS_CACHE_VERSION = "v1".freeze - WEEKLY_PROJECT_DIMENSION = "weekly_project".freeze - - private - - def filterable_dashboard_data - filters = dashboard_filters - interval = params[:interval] - key = [ current_user ] + filters.map { |field| params[field] } + [ interval.to_s, params[:from], params[:to] ] - - if dashboard_rollup_eligible? - build_filterable_dashboard_data(filters, interval) - else - Rails.cache.fetch(key, expires_in: 5.minutes) do - build_filterable_dashboard_data(filters, interval) - end - end - end - - def activity_graph_data - row = dashboard_rollup_fragment_row(DashboardRollup::ACTIVITY_GRAPH_DIMENSION) - return dashboard_activity_graph_from_rollup(row) if dashboard_activity_graph_rollup_valid?(row) - - dashboard_schedule_rollup_refresh(wait: 0.seconds) if dashboard_rollups_available? - dashboard_live_activity_graph_data - end - - def today_stats_data - row = dashboard_rollup_fragment_row(DashboardRollup::TODAY_STATS_DIMENSION) - return dashboard_today_stats_from_rollup(row) if dashboard_today_stats_rollup_valid?(row) - - dashboard_schedule_rollup_refresh(wait: 0.seconds) if dashboard_rollups_available? - dashboard_live_today_stats_data - end - - def dashboard_filters - %i[project language operating_system editor category] - end - - def build_filterable_dashboard_data(filters, interval) - archived = current_user.project_repo_mappings.archived.pluck(:project_name) - raw_filter_options = dashboard_raw_filter_options - result = dashboard_rollup_result(raw_filter_options, archived) - - result ||= dashboard_query_result(raw_filter_options, archived) - result[:selected_interval] = interval.to_s - result[:selected_from] = params[:from].to_s - result[:selected_to] = params[:to].to_s - filters.each { |field| result["selected_#{field}"] = params[field]&.split(",") || [] } - - result - end - - def dashboard_raw_filter_options - if dashboard_rollup_eligible? - rollup_options = dashboard_rollup_filter_options - return rollup_options if rollup_options - end - - dashboard_live_raw_filter_options - end - - def dashboard_live_raw_filter_options - cache_keys = dashboard_filters.index_with do |field| - "user_#{current_user.id}_dashboard_filter_options_#{field}_#{FILTER_OPTIONS_CACHE_VERSION}" - end - - reverse_lookup = cache_keys.invert - - cached = Rails.cache.fetch_multi(*cache_keys.values, expires_in: 15.minutes) do |cache_key| - current_user.heartbeats.distinct.pluck(reverse_lookup.fetch(cache_key)).compact_blank - end - - cache_keys.transform_values { |cache_key| cached.fetch(cache_key, []) } - end - - def dashboard_rollup_filter_options - return unless dashboard_rollups_available? - - row = dashboard_rollup_fragment_row(DashboardRollup::FILTER_OPTIONS_DIMENSION) - payload = row&.payload - unless payload.is_a?(Hash) && dashboard_filters.all? { |field| payload[field.to_s].is_a?(Array) || payload[field].is_a?(Array) } - dashboard_schedule_rollup_refresh(wait: 0.seconds) - return - end - - dashboard_filters.index_with do |field| - Array(payload[field.to_s] || payload[field]) - end - end - - def dashboard_query_result(raw_filter_options, archived) - hb = current_user.heartbeats - result = dashboard_filter_options_result(raw_filter_options, archived) - - Time.use_zone(current_user.timezone) do - dashboard_filters.each do |field| - next unless params[field].present? - - arr = params[field].split(",") - hb = if %i[operating_system editor].include?(field) - hb.where(field => arr.flat_map { |value| [ value.downcase, value.capitalize ] }.uniq) - elsif field == :language - raw = raw_filter_options.fetch(:language, []).select { |language| arr.include?(language.categorize_language) } - raw.any? ? hb.where(field => raw) : hb - else - hb.where(field => arr) - end - result["singular_#{field}"] = arr.length == 1 - end - - hb = hb.filter_by_time_range(params[:interval], params[:from], params[:to]) - dashboard_fill_aggregate_result( - result: result, - grouped_durations: dashboard_grouped_durations_snapshot(hb), - total_time: hb.duration_seconds, - total_heartbeats: hb.count, - weekly_project_stats: dashboard_weekly_project_stats(hb, current_user.timezone), - archived: archived - ) - end - - result - end - - def dashboard_rollup_result(raw_filter_options, archived) - snapshot = dashboard_aggregate_rollup_snapshot - return unless snapshot - - result = dashboard_filter_options_result(raw_filter_options, archived) - - Time.use_zone(current_user.timezone) do - dashboard_fill_aggregate_result( - result: result, - grouped_durations: snapshot.fetch(:grouped_durations), - total_time: snapshot.fetch(:total_time), - total_heartbeats: snapshot.fetch(:total_heartbeats), - weekly_project_stats: snapshot.fetch(:weekly_project_stats), - archived: archived - ) - end - - result - end - - def dashboard_filter_options_result(raw_filter_options, archived) - h = ApplicationController.helpers - - dashboard_filters.each_with_object({}) do |field, result| - options = raw_filter_options.fetch(field, []) - options = options.reject { |name| archived.include?(name) } if field == :project - result[field] = options.map { |value| - if field == :language then value.categorize_language - elsif field == :editor then h.display_editor_name(value) - elsif field == :operating_system then h.display_os_name(value) - else value - end - }.uniq - end - end - - def dashboard_fill_aggregate_result(result:, grouped_durations:, total_time:, total_heartbeats:, weekly_project_stats:, archived:) - h = ApplicationController.helpers - - result[:total_time] = total_time - result[:total_heartbeats] = total_heartbeats - - dashboard_filters.each do |field| - stats = dashboard_grouped_durations(grouped_durations, field, archived) - result["top_#{field}"] = stats.max_by { |_, duration| duration }&.first - end - - result["top_editor"] &&= h.display_editor_name(result["top_editor"]) - result["top_operating_system"] &&= h.display_os_name(result["top_operating_system"]) - result["top_language"] &&= h.display_language_name(result["top_language"]) - - unless result["singular_project"] - result[:project_durations] = dashboard_grouped_durations(grouped_durations, :project, archived) - .sort_by { |_, duration| -duration }.first(10).to_h - end - - %i[language editor operating_system category].each do |field| - next if result["singular_#{field}"] - - stats = grouped_durations.fetch(field, {}).each_with_object({}) do |(raw, duration), agg| - key = raw.to_s.presence || "Unknown" - key = if field == :language - key == "Unknown" ? key : key.categorize_language - elsif %i[editor operating_system].include?(field) - key.downcase - else - key - end - agg[key] = (agg[key] || 0) + duration - end - - result["#{field}_stats"] = stats.sort_by { |_, duration| -duration }.first(10).map { |key, value| - label = case field - when :editor then h.display_editor_name(key) - when :operating_system then h.display_os_name(key) - when :language then h.display_language_name(key) - else key - end - [ label, value ] - }.to_h - end - - if result["language_stats"].present? - result[:language_colors] = LanguageUtils.colors_for(result["language_stats"].keys) - end - - result[:weekly_project_stats] = weekly_project_stats.transform_values do |stats| - stats.reject { |project, _| archived.include?(project) } - end - end - - def dashboard_aggregate_rollup_snapshot - return unless dashboard_rollups_available? - return unless dashboard_rollup_eligible? - - total_row = dashboard_rollup_total_row - unless total_row - dashboard_schedule_rollup_refresh(wait: 0.seconds) - return - end - - dashboard_schedule_rollup_refresh(wait: 0.seconds) if dashboard_aggregate_rollup_stale?(total_row) - - { - total_time: total_row.total_seconds, - total_heartbeats: total_row.source_heartbeats_count.to_i, - grouped_durations: dashboard_filters.index_with do |field| - dashboard_rollup_rows_by_dimension.fetch(field.to_s, []).to_h { |row| [ row.bucket, row.total_seconds ] } - end, - weekly_project_stats: dashboard_rollup_weekly_project_stats( - dashboard_rollup_rows_by_dimension.fetch(WEEKLY_PROJECT_DIMENSION, []) - ) - } - end - - def dashboard_rollup_eligible? - params[:interval].blank? && - params[:from].blank? && - params[:to].blank? && - dashboard_filters.none? { |field| params[field].present? } - end - - def dashboard_rollups_available? - DashboardRollup.table_exists? - rescue ActiveRecord::StatementInvalid - false - end - - def dashboard_rollup_rows - return [] unless dashboard_rollups_available? - - @dashboard_rollup_rows ||= DashboardRollup.where(user_id: current_user.id).to_a - end - - def dashboard_rollup_rows_by_dimension - @dashboard_rollup_rows_by_dimension ||= dashboard_rollup_rows.group_by(&:dimension) - end - - def dashboard_rollup_fragment_row(dimension) - dashboard_rollup_rows_by_dimension.fetch(dimension.to_s, []).first - end - - def dashboard_rollup_total_row - @dashboard_rollup_total_row ||= dashboard_rollup_rows.find(&:total_dimension?) - end - - def dashboard_aggregate_rollup_stale?(total_row) - DashboardRollup.dirty?(current_user.id) || - dashboard_rollup_time_fingerprint(total_row.source_max_heartbeat_time) != dashboard_rollup_source_max_heartbeat_time - end - - def dashboard_schedule_rollup_refresh(wait:) - return if @dashboard_rollup_refresh_scheduled - - DashboardRollupRefreshJob.schedule_for(current_user.id, wait: wait) - @dashboard_rollup_refresh_scheduled = true - end - - def dashboard_activity_graph_rollup_valid?(row) - payload = row&.payload - return false unless payload.is_a?(Hash) - - start_date, end_date = dashboard_activity_graph_date_range(current_user.timezone) - payload["timezone"] == current_user.timezone && - payload["start_date"] == start_date && - payload["end_date"] == end_date && - payload["duration_by_date"].is_a?(Hash) - end - - def dashboard_today_stats_rollup_valid?(row) - payload = row&.payload - return false unless payload.is_a?(Hash) - - payload["timezone"] == current_user.timezone && - payload["today_date"] == dashboard_today_date && - payload.key?("todays_duration_seconds") && - payload["todays_language_categories"].is_a?(Array) && - payload["todays_editor_keys"].is_a?(Array) - end - - def dashboard_live_activity_graph_data - timezone = current_user.timezone - start_date, end_date = dashboard_activity_graph_date_range(timezone) - durations = Rails.cache.fetch(current_user.activity_graph_cache_key(timezone), expires_in: 1.minute) do - Time.use_zone(timezone) { current_user.heartbeats.daily_durations(user_timezone: timezone).to_h } - end - - dashboard_activity_graph_result( - start_date: start_date, - end_date: end_date, - duration_by_date: durations, - timezone: timezone - ) - end - - def dashboard_activity_graph_from_rollup(row) - payload = row.payload || {} - - dashboard_activity_graph_result( - start_date: payload["start_date"], - end_date: payload["end_date"], - duration_by_date: payload["duration_by_date"], - timezone: payload["timezone"] || current_user.timezone - ) - end - - def dashboard_activity_graph_result(start_date:, end_date:, duration_by_date:, timezone:) - { - start_date: start_date, - end_date: end_date, - duration_by_date: duration_by_date.to_h.transform_keys { |date| date.to_s }.transform_values(&:to_i), - busiest_day_seconds: 8.hours.to_i, - timezone_label: ActiveSupport::TimeZone[timezone]&.to_s || timezone - } - end - - def dashboard_live_today_stats_data - snapshot = dashboard_today_stats_snapshot(current_user.heartbeats) - dashboard_today_stats_result( - todays_duration_seconds: snapshot[:todays_duration_seconds], - todays_language_categories: snapshot[:todays_language_categories], - todays_editor_keys: snapshot[:todays_editor_keys] - ) - end - - def dashboard_today_stats_from_rollup(row) - payload = row.payload || {} - - dashboard_today_stats_result( - todays_duration_seconds: payload["todays_duration_seconds"], - todays_language_categories: payload["todays_language_categories"], - todays_editor_keys: payload["todays_editor_keys"] - ) - end - - def dashboard_today_stats_result(todays_duration_seconds:, todays_language_categories:, todays_editor_keys:) - h = ApplicationController.helpers - todays_languages = Array(todays_language_categories).filter_map do |language| - h.display_language_name(language) if language.present? - end - todays_editors = Array(todays_editor_keys).filter_map do |editor| - h.display_editor_name(editor) if editor.present? - end - todays_duration = todays_duration_seconds.to_i - show_logged_time_sentence = todays_duration > 1.minute && (todays_languages.any? || todays_editors.any?) - - { - show_logged_time_sentence: show_logged_time_sentence, - todays_duration_display: h.short_time_detailed(todays_duration), - todays_languages: todays_languages, - todays_editors: todays_editors - } - end - - def dashboard_today_stats_snapshot(scope) - Time.use_zone(current_user.timezone) do - rows = scope.today - .select(:language, :editor, - "COUNT(*) OVER (PARTITION BY language) as language_count", - "COUNT(*) OVER (PARTITION BY editor) as editor_count") - .distinct.to_a - - language_categories = rows - .map { |row| [ row.language&.categorize_language, row.language_count ] } - .reject { |language, _| language.blank? } - .group_by(&:first) - .transform_values { |pairs| pairs.sum(&:last) } - .sort_by { |_, count| -count } - .map(&:first) - - editor_keys = rows - .map { |row| [ row.editor, row.editor_count ] } - .reject { |editor, _| editor.blank? } - .uniq - .sort_by { |_, count| -count } - .map(&:first) - - { - today_date: Date.current.iso8601, - todays_duration_seconds: scope.today.duration_seconds.to_i, - todays_language_categories: language_categories, - todays_editor_keys: editor_keys - } - end - end - - def dashboard_today_date - Time.use_zone(current_user.timezone) { Date.current.iso8601 } - end - - def dashboard_activity_graph_date_range(timezone) - Time.use_zone(timezone) do - [ 365.days.ago.to_date.iso8601, Date.current.iso8601 ] - end - end - - def dashboard_rollup_source_max_heartbeat_time - dashboard_rollup_time_fingerprint(current_user.heartbeats.maximum(:time)) - end - - def dashboard_rollup_weekly_project_stats(rows) - result = dashboard_week_ranges.to_h { |week_key, *_| [ week_key, {} ] } - - rows.each do |row| - week_key, project = JSON.parse(row.bucket_value) - next unless result.key?(week_key) - - result[week_key][project] = row.total_seconds - end - - result - end - - def dashboard_rollup_time_fingerprint(timestamp) - return if timestamp.nil? - - (timestamp * 1_000_000).round - end - - def dashboard_grouped_durations_snapshot(scope) - dashboard_filters.index_with do |field| - field == :project ? dashboard_project_grouped_durations(scope) : scope.group(field).duration_seconds - end - end - - def dashboard_grouped_durations(grouped_durations, field, archived) - stats = grouped_durations.fetch(field, {}) - return stats unless field == :project - - stats.reject { |project, _| archived.include?(project) } - end - - def dashboard_project_grouped_durations(scope) - non_null = scope.where.not(project: nil).group(:project).duration_seconds - return non_null if scope.where(project: nil).none? - - null_duration = scope.where(project: nil).duration_seconds - return non_null if null_duration.zero? - - non_null.merge(nil => null_duration) - end - - def dashboard_weekly_project_stats(scope, timezone) - week_ranges = dashboard_week_ranges - result = week_ranges.to_h { |week_key, *_| [ week_key, {} ] } - - relation_sql = scope.with_valid_timestamps - .where.not(time: nil) - .where(time: week_ranges.last[1]..week_ranges.first[2]) - .select(:time, :project) - .to_sql - - quoted_timezone = Heartbeat.connection.quote(timezone) - week_group_sql = "DATE_TRUNC('week', to_timestamp(time) AT TIME ZONE #{quoted_timezone})" - - rows = Heartbeat.connection.select_all(<<~SQL.squish) - SELECT TO_CHAR(week_group, 'YYYY-MM-DD') AS week_key, - grouped_time, - COALESCE(SUM(diff), 0)::integer AS duration - FROM ( - SELECT project AS grouped_time, - #{week_group_sql} AS week_group, - CASE - WHEN LAG(time) OVER (PARTITION BY project, #{week_group_sql} ORDER BY time) IS NULL THEN 0 - ELSE LEAST( - time - LAG(time) OVER (PARTITION BY project, #{week_group_sql} ORDER BY time), - #{Heartbeat.heartbeat_timeout_duration.to_i} - ) - END AS diff - FROM (#{relation_sql}) dashboard_heartbeats - ) diffs - GROUP BY week_group, grouped_time - ORDER BY week_key DESC, grouped_time - SQL - - rows.each do |row| - result[row["week_key"]][row["grouped_time"]] = row["duration"].to_i - end - - result - end - - def dashboard_week_ranges - Time.use_zone(current_user.timezone) do - (0..11).map do |week_offset| - week_start = week_offset.weeks.ago.beginning_of_week - [ week_start.to_date.iso8601, week_start.to_f, week_offset.weeks.ago.end_of_week.to_f ] - end - end - end -end diff --git a/app/controllers/static_pages_controller.rb b/app/controllers/static_pages_controller.rb index 16f78074f..46def5ca6 100644 --- a/app/controllers/static_pages_controller.rb +++ b/app/controllers/static_pages_controller.rb @@ -1,6 +1,4 @@ class StaticPagesController < InertiaController - include DashboardData - layout "inertia", only: %i[index wakatime_alternative] def index @@ -146,14 +144,20 @@ def signed_in_props end def dashboard_stats_payload + snapshot = dashboard_snapshot + { - filterable_dashboard_data: filterable_dashboard_data, - activity_graph: activity_graph_data, - today_stats: today_stats_data, + filterable_dashboard_data: snapshot.filterable_dashboard_data, + activity_graph: snapshot.activity_graph, + today_stats: snapshot.today_stats, programming_goals_progress: programming_goals_progress_data } end + def dashboard_snapshot + @dashboard_snapshot ||= DashboardSnapshot.new(user: current_user, params: params, rollup_rows: @dashboard_rollup_rows).call + end + def signed_out_props { flavor_text: @flavor_text.to_s, @@ -181,17 +185,17 @@ def programming_goals_progress_data end def initial_dashboard_stats_prop - return unless dashboard_rollup_eligible? - return unless dashboard_rollups_available? + return unless params[:interval].blank? && params[:from].blank? && params[:to].blank? + return if DashboardSnapshot::FILTERS.any? { |field| params[field].present? } + return unless DashboardRollup.table_exists? rows = DashboardRollup.where(user_id: current_user.id).to_a - total_row = rows.find(&:total_dimension?) - return unless total_row + return unless rows.any?(&:total_dimension?) @dashboard_rollup_rows = rows - @dashboard_rollup_rows_by_dimension = rows.group_by(&:dimension) - @dashboard_rollup_total_row = total_row dashboard_stats_payload + rescue ActiveRecord::StatementInvalid + nil end end diff --git a/app/jobs/dashboard_rollup_refresh_job.rb b/app/jobs/dashboard_rollup_refresh_job.rb index 3c3bc55e3..8ca6c0687 100644 --- a/app/jobs/dashboard_rollup_refresh_job.rb +++ b/app/jobs/dashboard_rollup_refresh_job.rb @@ -33,7 +33,7 @@ def perform(user_id) user = User.find_by(id: user_id) return unless user - DashboardRollupRefreshService.new(user: user).call + DashboardSnapshot.new(user: user).persist_rollups! ensure Rails.cache.delete(self.class.enqueue_cache_key(user_id)) end diff --git a/app/services/dashboard_rollup_refresh_service.rb b/app/services/dashboard_rollup_refresh_service.rb deleted file mode 100644 index c5a2a9398..000000000 --- a/app/services/dashboard_rollup_refresh_service.rb +++ /dev/null @@ -1,224 +0,0 @@ -class DashboardRollupRefreshService < ApplicationService - GROUPED_DIMENSIONS = %i[project language editor operating_system category].freeze - WEEKLY_PROJECT_DIMENSION = "weekly_project".freeze - - def initialize(user:) - @user = user - @scope = user.heartbeats - end - - def call - now = Time.current - records = [ - build_total_record(now), - build_filter_options_record(now), - build_activity_graph_record(now), - build_today_stats_record(now) - ] - - GROUPED_DIMENSIONS.each do |dimension| - grouped_durations(dimension).each do |bucket, total_seconds| - records << build_record( - dimension: dimension, - bucket: bucket, - total_seconds: total_seconds, - now: now - ) - end - end - weekly_project_stats.each do |week_key, projects| - projects.each do |project, total_seconds| - records << build_record( - dimension: WEEKLY_PROJECT_DIMENSION, - bucket: [ week_key, project ].to_json, - total_seconds: total_seconds, - now: now - ) - end - end - - DashboardRollup.transaction do - DashboardRollup.where(user_id: @user.id).delete_all - DashboardRollup.insert_all!(records) - end - - DashboardRollup.clear_dirty(@user.id) - end - - private - - def build_total_record(now) - build_record( - dimension: DashboardRollup::TOTAL_DIMENSION, - bucket: nil, - total_seconds: @scope.duration_seconds, - now: now, - source_heartbeats_count: @scope.count, - source_max_heartbeat_time: @scope.maximum(:time) - ) - end - - def build_activity_graph_record(now) - build_record( - dimension: DashboardRollup::ACTIVITY_GRAPH_DIMENSION, - bucket: nil, - total_seconds: 0, - now: now, - payload: activity_graph_payload - ) - end - - def build_filter_options_record(now) - build_record( - dimension: DashboardRollup::FILTER_OPTIONS_DIMENSION, - bucket: nil, - total_seconds: 0, - now: now, - payload: filter_options_payload - ) - end - - def build_today_stats_record(now) - build_record( - dimension: DashboardRollup::TODAY_STATS_DIMENSION, - bucket: nil, - total_seconds: 0, - now: now, - payload: today_stats_payload - ) - end - - def build_record(dimension:, bucket:, total_seconds:, now:, source_heartbeats_count: nil, source_max_heartbeat_time: nil, payload: nil) - { - user_id: @user.id, - dimension: dimension.to_s, - bucket_value: bucket.to_s, - bucket_value_present: !bucket.nil?, - total_seconds: total_seconds.to_i, - source_heartbeats_count: source_heartbeats_count, - source_max_heartbeat_time: source_max_heartbeat_time, - payload: payload, - created_at: now, - updated_at: now - } - end - - def grouped_durations(dimension) - return project_grouped_durations if dimension == :project - - @scope.group(dimension).duration_seconds - end - - def project_grouped_durations - non_null = @scope.where.not(project: nil).group(:project).duration_seconds - return non_null if @scope.where(project: nil).none? - - null_duration = @scope.where(project: nil).duration_seconds - return non_null if null_duration.zero? - - non_null.merge(nil => null_duration) - end - - def weekly_project_stats - week_ranges = dashboard_week_ranges - result = week_ranges.to_h { |week_key, *_| [ week_key, {} ] } - - relation_sql = @scope.with_valid_timestamps - .where.not(time: nil) - .where(time: week_ranges.last[1]..week_ranges.first[2]) - .select(:time, :project) - .to_sql - - quoted_timezone = Heartbeat.connection.quote(@user.timezone) - week_group_sql = "DATE_TRUNC('week', to_timestamp(time) AT TIME ZONE #{quoted_timezone})" - - rows = Heartbeat.connection.select_all(<<~SQL.squish) - SELECT TO_CHAR(week_group, 'YYYY-MM-DD') AS week_key, - grouped_time, - COALESCE(SUM(diff), 0)::integer AS duration - FROM ( - SELECT project AS grouped_time, - #{week_group_sql} AS week_group, - CASE - WHEN LAG(time) OVER (PARTITION BY project, #{week_group_sql} ORDER BY time) IS NULL THEN 0 - ELSE LEAST( - time - LAG(time) OVER (PARTITION BY project, #{week_group_sql} ORDER BY time), - #{Heartbeat.heartbeat_timeout_duration.to_i} - ) - END AS diff - FROM (#{relation_sql}) dashboard_heartbeats - ) diffs - GROUP BY week_group, grouped_time - ORDER BY week_key DESC, grouped_time - SQL - - rows.each do |row| - result[row["week_key"]][row["grouped_time"]] = row["duration"].to_i - end - - result - end - - def dashboard_week_ranges - Time.use_zone(@user.timezone) do - (0..11).map do |w| - week_start = w.weeks.ago.beginning_of_week - [ week_start.to_date.iso8601, week_start.to_f, w.weeks.ago.end_of_week.to_f ] - end - end - end - - def activity_graph_payload - Time.use_zone(@user.timezone) do - end_date = Date.current - start_date = 365.days.ago.to_date - durations = @scope.daily_durations(user_timezone: @user.timezone).to_h - - { - timezone: @user.timezone, - start_date: start_date.iso8601, - end_date: end_date.iso8601, - duration_by_date: durations.transform_keys { |date| date.to_date.iso8601 }.transform_values(&:to_i) - } - end - end - - def filter_options_payload - GROUPED_DIMENSIONS.index_with do |dimension| - @scope.distinct.pluck(dimension).compact_blank.sort - end - end - - def today_stats_payload - Time.use_zone(@user.timezone) do - rows = @scope.today - .select(:language, :editor, - "COUNT(*) OVER (PARTITION BY language) as language_count", - "COUNT(*) OVER (PARTITION BY editor) as editor_count") - .distinct.to_a - - language_categories = rows - .map { |row| [ row.language&.categorize_language, row.language_count ] } - .reject { |language, _| language.blank? } - .group_by(&:first) - .transform_values { |pairs| pairs.sum(&:last) } - .sort_by { |_, count| -count } - .map(&:first) - - editor_keys = rows - .map { |row| [ row.editor, row.editor_count ] } - .reject { |editor, _| editor.blank? } - .uniq - .sort_by { |_, count| -count } - .map(&:first) - - { - timezone: @user.timezone, - today_date: Date.current.iso8601, - todays_duration_seconds: @scope.today.duration_seconds.to_i, - todays_language_categories: language_categories, - todays_editor_keys: editor_keys - } - end - end -end diff --git a/app/services/dashboard_snapshot.rb b/app/services/dashboard_snapshot.rb new file mode 100644 index 000000000..af2906a5d --- /dev/null +++ b/app/services/dashboard_snapshot.rb @@ -0,0 +1,550 @@ +class DashboardSnapshot < ApplicationService + FILTERS = %i[project language operating_system editor category].freeze + FILTER_OPTIONS_CACHE_VERSION = "v1".freeze + GROUPED_DIMENSIONS = FILTERS.freeze + WEEKLY_PROJECT_DIMENSION = "weekly_project".freeze + + Result = Data.define(:filterable_dashboard_data, :activity_graph, :today_stats, :sources) + Raw = Data.define( + :total_time, + :total_heartbeats, + :source_max_heartbeat_time, + :grouped_durations, + :filter_options, + :weekly_project_stats, + :activity_graph, + :today_stats, + :timezone + ) + + def initialize(user:, params: {}, rollup_rows: nil) + @user = user + @params = params.respond_to?(:to_unsafe_h) ? params : ActionController::Parameters.new(params) + @rollup_rows = rollup_rows + @sources = {} + @rollup_refresh_scheduled = false + end + + def call + raw_filter_options = raw_filter_options_for_read + archived = @user.project_repo_mappings.archived.pluck(:project_name) + aggregate_raw = aggregate_raw_for_read(raw_filter_options) + + Result.new( + filterable_dashboard_data: build_filterable_dashboard_data(aggregate_raw, archived), + activity_graph: build_activity_graph(activity_graph_raw_for_read), + today_stats: build_today_stats(today_stats_raw_for_read), + sources: @sources.dup + ) + end + + def persist_rollups! + now = Time.current + raw = live_raw_snapshot(filter_options: live_raw_filter_options).with( + activity_graph: live_activity_graph_raw, + today_stats: live_today_stats_raw(@user.heartbeats) + ) + records = rollup_records(raw, now: now) + + DashboardRollup.transaction do + DashboardRollup.where(user_id: @user.id).delete_all + DashboardRollup.insert_all!(records) + end + + DashboardRollup.clear_dirty(@user.id) + end + + private + + def build_filterable_dashboard_data(raw, archived) + result = filter_options_result(raw.filter_options, archived) + filter_selections = selected_filters + + result[:total_time] = raw.total_time + result[:total_heartbeats] = raw.total_heartbeats + result[:selected_interval] = @params[:interval].to_s + result[:selected_from] = @params[:from].to_s + result[:selected_to] = @params[:to].to_s + + FILTERS.each do |field| + arr = filter_selections.fetch(field) + result["selected_#{field}"] = arr + result["singular_#{field}"] = arr.length == 1 if arr.any? + result["top_#{field}"] = grouped_durations(raw.grouped_durations, field, archived).max_by { |_, duration| duration }&.first + end + + h = ApplicationController.helpers + result["top_editor"] &&= h.display_editor_name(result["top_editor"]) + result["top_operating_system"] &&= h.display_os_name(result["top_operating_system"]) + result["top_language"] &&= h.display_language_name(result["top_language"]) + + unless result["singular_project"] + result[:project_durations] = grouped_durations(raw.grouped_durations, :project, archived) + .sort_by { |_, duration| -duration }.first(10).to_h + end + + %i[language editor operating_system category].each do |field| + next if result["singular_#{field}"] + + stats = raw.grouped_durations.fetch(field, {}).each_with_object({}) do |(raw_key, duration), agg| + key = raw_key.to_s.presence || "Unknown" + key = if field == :language + key == "Unknown" ? key : key.categorize_language + elsif %i[editor operating_system].include?(field) + key.downcase + else + key + end + agg[key] = (agg[key] || 0) + duration + end + + result["#{field}_stats"] = stats.sort_by { |_, duration| -duration }.first(10).map { |key, value| + label = case field + when :editor then h.display_editor_name(key) + when :operating_system then h.display_os_name(key) + when :language then h.display_language_name(key) + else key + end + [ label, value ] + }.to_h + end + + result[:language_colors] = LanguageUtils.colors_for(result["language_stats"].keys) if result["language_stats"].present? + result[:weekly_project_stats] = raw.weekly_project_stats.transform_values { |stats| stats.reject { |project, _| archived.include?(project) } } + result + end + + def build_activity_graph(raw) + { + start_date: raw.fetch(:start_date), + end_date: raw.fetch(:end_date), + duration_by_date: raw.fetch(:duration_by_date).to_h.transform_keys { |date| date.to_s }.transform_values(&:to_i), + busiest_day_seconds: 8.hours.to_i, + timezone_label: ActiveSupport::TimeZone[raw.fetch(:timezone)]&.to_s || raw.fetch(:timezone) + } + end + + def build_today_stats(raw) + h = ApplicationController.helpers + todays_languages = Array(raw.fetch(:todays_language_categories)).filter_map { |language| h.display_language_name(language) if language.present? } + todays_editors = Array(raw.fetch(:todays_editor_keys)).filter_map { |editor| h.display_editor_name(editor) if editor.present? } + todays_duration = raw.fetch(:todays_duration_seconds).to_i + + { + show_logged_time_sentence: todays_duration > 1.minute && (todays_languages.any? || todays_editors.any?), + todays_duration_display: h.short_time_detailed(todays_duration), + todays_languages: todays_languages, + todays_editors: todays_editors + } + end + + def aggregate_raw_for_read(raw_filter_options) + if rollup_eligible? + raw = aggregate_rollup_raw(raw_filter_options) + return raw if raw + end + + @sources[:aggregate] = :live + scope = filtered_scope(raw_filter_options) + if rollup_eligible? + live_raw_snapshot(filter_options: raw_filter_options, scope: scope) + else + cached = Rails.cache.fetch(live_aggregate_cache_key, expires_in: 5.minutes) do + live_raw_snapshot(filter_options: raw_filter_options, scope: scope).to_h + end + cached.is_a?(Raw) ? cached : Raw.new(**cached.symbolize_keys) + end + end + + def raw_filter_options_for_read + if rollup_eligible? + options = rollup_filter_options + return options if options + end + + @sources[:filter_options] = :live + live_raw_filter_options + end + + def aggregate_rollup_raw(filter_options) + return unless rollups_available? + return unless rollup_eligible? + + total_row = rollup_total_row + unless total_row + schedule_rollup_refresh(wait: 0.seconds) + return + end + + if aggregate_rollup_stale?(total_row) + schedule_rollup_refresh(wait: 0.seconds) + @sources[:aggregate] = :stale_rollup + else + @sources[:aggregate] = :rollup + end + + Raw.new( + total_time: total_row.total_seconds, + total_heartbeats: total_row.source_heartbeats_count.to_i, + source_max_heartbeat_time: total_row.source_max_heartbeat_time, + grouped_durations: FILTERS.index_with { |field| rollup_rows_by_dimension.fetch(field.to_s, []).to_h { |row| [ row.bucket, row.total_seconds ] } }, + filter_options: filter_options, + weekly_project_stats: rollup_weekly_project_stats(rollup_rows_by_dimension.fetch(WEEKLY_PROJECT_DIMENSION, [])), + activity_graph: nil, + today_stats: nil, + timezone: @user.timezone + ) + end + + def activity_graph_raw_for_read + row = rollup_fragment_row(DashboardRollup::ACTIVITY_GRAPH_DIMENSION) + if activity_graph_rollup_valid?(row) + @sources[:activity_graph] = :rollup + return activity_graph_from_rollup(row) + end + + schedule_rollup_refresh(wait: 0.seconds) if rollups_available? + @sources[:activity_graph] = :live + live_activity_graph_raw + end + + def today_stats_raw_for_read + row = rollup_fragment_row(DashboardRollup::TODAY_STATS_DIMENSION) + if today_stats_rollup_valid?(row) + @sources[:today_stats] = :rollup + return today_stats_from_rollup(row) + end + + schedule_rollup_refresh(wait: 0.seconds) if rollups_available? + @sources[:today_stats] = :live + live_today_stats_raw(@user.heartbeats) + end + + def live_raw_snapshot(filter_options:, scope: @user.heartbeats) + Time.use_zone(@user.timezone) do + Raw.new( + total_time: scope.duration_seconds, + total_heartbeats: scope.count, + source_max_heartbeat_time: scope.maximum(:time), + grouped_durations: grouped_durations_snapshot(scope), + filter_options: filter_options, + weekly_project_stats: weekly_project_stats(scope, @user.timezone), + activity_graph: nil, + today_stats: nil, + timezone: @user.timezone + ) + end + end + + def live_raw_filter_options + cache_keys = FILTERS.index_with { |field| "user_#{@user.id}_dashboard_filter_options_#{field}_#{FILTER_OPTIONS_CACHE_VERSION}" } + reverse_lookup = cache_keys.invert + + cached = Rails.cache.fetch_multi(*cache_keys.values, expires_in: 15.minutes) do |cache_key| + @user.heartbeats.distinct.pluck(reverse_lookup.fetch(cache_key)).compact_blank + end + + cache_keys.transform_values { |cache_key| cached.fetch(cache_key, []) } + end + + def rollup_filter_options + return unless rollups_available? + + row = rollup_fragment_row(DashboardRollup::FILTER_OPTIONS_DIMENSION) + payload = row&.payload + unless payload.is_a?(Hash) && FILTERS.all? { |field| payload[field.to_s].is_a?(Array) || payload[field].is_a?(Array) } + schedule_rollup_refresh(wait: 0.seconds) + return + end + + @sources[:filter_options] = :rollup + FILTERS.index_with { |field| Array(payload[field.to_s] || payload[field]) } + end + + def filtered_scope(raw_filter_options) + hb = @user.heartbeats + selected_filters.each do |field, arr| + next if arr.blank? + + hb = if %i[operating_system editor].include?(field) + hb.where(field => arr.flat_map { |value| [ value.downcase, value.capitalize ] }.uniq) + elsif field == :language + raw = raw_filter_options.fetch(:language, []).select { |language| arr.include?(language.categorize_language) } + raw.any? ? hb.where(field => raw) : hb + else + hb.where(field => arr) + end + end + + hb.filter_by_time_range(@params[:interval], @params[:from], @params[:to]) + end + + def selected_filters + FILTERS.index_with { |field| @params[field]&.split(",") || [] } + end + + def live_aggregate_cache_key + [ @user ] + FILTERS.map { |field| @params[field] } + [ @params[:interval].to_s, @params[:from], @params[:to] ] + end + + def filter_options_result(raw_filter_options, archived) + h = ApplicationController.helpers + + FILTERS.each_with_object({}) do |field, result| + options = raw_filter_options.fetch(field, []) + options = options.reject { |name| archived.include?(name) } if field == :project + result[field] = options.map { |value| + if field == :language then value.categorize_language + elsif field == :editor then h.display_editor_name(value) + elsif field == :operating_system then h.display_os_name(value) + else value + end + }.uniq + end + end + + def live_activity_graph_raw + timezone = @user.timezone + start_date, end_date = activity_graph_date_range(timezone) + durations = Rails.cache.fetch(@user.activity_graph_cache_key(timezone), expires_in: 1.minute) do + Time.use_zone(timezone) { @user.heartbeats.daily_durations(user_timezone: timezone).to_h } + end + + { timezone: timezone, start_date: start_date, end_date: end_date, duration_by_date: durations } + end + + def activity_graph_from_rollup(row) + payload = row.payload || {} + { + timezone: payload["timezone"] || @user.timezone, + start_date: payload["start_date"], + end_date: payload["end_date"], + duration_by_date: payload["duration_by_date"] + } + end + + def live_today_stats_raw(scope) + Time.use_zone(@user.timezone) do + rows = scope.today + .select(:language, :editor, + "COUNT(*) OVER (PARTITION BY language) as language_count", + "COUNT(*) OVER (PARTITION BY editor) as editor_count") + .distinct.to_a + + language_categories = rows + .map { |row| [ row.language&.categorize_language, row.language_count ] } + .reject { |language, _| language.blank? } + .group_by(&:first) + .transform_values { |pairs| pairs.sum(&:last) } + .sort_by { |_, count| -count } + .map(&:first) + + editor_keys = rows + .map { |row| [ row.editor, row.editor_count ] } + .reject { |editor, _| editor.blank? } + .uniq + .sort_by { |_, count| -count } + .map(&:first) + + { + timezone: @user.timezone, + today_date: Date.current.iso8601, + todays_duration_seconds: scope.today.duration_seconds.to_i, + todays_language_categories: language_categories, + todays_editor_keys: editor_keys + } + end + end + + def today_stats_from_rollup(row) + payload = row.payload || {} + { + timezone: payload["timezone"] || @user.timezone, + today_date: payload["today_date"], + todays_duration_seconds: payload["todays_duration_seconds"], + todays_language_categories: payload["todays_language_categories"], + todays_editor_keys: payload["todays_editor_keys"] + } + end + + def grouped_durations_snapshot(scope) + FILTERS.index_with { |field| field == :project ? project_grouped_durations(scope) : scope.group(field).duration_seconds } + end + + def project_grouped_durations(scope) + non_null = scope.where.not(project: nil).group(:project).duration_seconds + return non_null if scope.where(project: nil).none? + + null_duration = scope.where(project: nil).duration_seconds + return non_null if null_duration.zero? + + non_null.merge(nil => null_duration) + end + + def grouped_durations(grouped, field, archived) + stats = grouped.fetch(field, {}) + field == :project ? stats.reject { |project, _| archived.include?(project) } : stats + end + + def weekly_project_stats(scope, timezone) + week_ranges = week_ranges() + result = week_ranges.to_h { |week_key, *_| [ week_key, {} ] } + + relation_sql = scope.with_valid_timestamps.where.not(time: nil).where(time: week_ranges.last[1]..week_ranges.first[2]).select(:time, :project).to_sql + quoted_timezone = Heartbeat.connection.quote(timezone) + week_group_sql = "DATE_TRUNC('week', to_timestamp(time) AT TIME ZONE #{quoted_timezone})" + + rows = Heartbeat.connection.select_all(<<~SQL.squish) + SELECT TO_CHAR(week_group, 'YYYY-MM-DD') AS week_key, + grouped_time, + COALESCE(SUM(diff), 0)::integer AS duration + FROM ( + SELECT project AS grouped_time, + #{week_group_sql} AS week_group, + CASE + WHEN LAG(time) OVER (PARTITION BY project, #{week_group_sql} ORDER BY time) IS NULL THEN 0 + ELSE LEAST(time - LAG(time) OVER (PARTITION BY project, #{week_group_sql} ORDER BY time), #{Heartbeat.heartbeat_timeout_duration.to_i}) + END AS diff + FROM (#{relation_sql}) dashboard_heartbeats + ) diffs + GROUP BY week_group, grouped_time + ORDER BY week_key DESC, grouped_time + SQL + + rows.each { |row| result[row["week_key"]][row["grouped_time"]] = row["duration"].to_i } + result + end + + def rollup_records(raw, now:) + records = [ + build_record(dimension: DashboardRollup::TOTAL_DIMENSION, bucket: nil, total_seconds: raw.total_time, now: now, source_heartbeats_count: raw.total_heartbeats, source_max_heartbeat_time: raw.source_max_heartbeat_time), + build_record(dimension: DashboardRollup::FILTER_OPTIONS_DIMENSION, bucket: nil, total_seconds: 0, now: now, payload: raw.filter_options), + build_record(dimension: DashboardRollup::ACTIVITY_GRAPH_DIMENSION, bucket: nil, total_seconds: 0, now: now, payload: raw.activity_graph), + build_record(dimension: DashboardRollup::TODAY_STATS_DIMENSION, bucket: nil, total_seconds: 0, now: now, payload: raw.today_stats) + ] + + GROUPED_DIMENSIONS.each do |dimension| + raw.grouped_durations.fetch(dimension, {}).each do |bucket, total_seconds| + records << build_record(dimension: dimension, bucket: bucket, total_seconds: total_seconds, now: now) + end + end + + raw.weekly_project_stats.each do |week_key, projects| + projects.each do |project, total_seconds| + records << build_record(dimension: WEEKLY_PROJECT_DIMENSION, bucket: [ week_key, project ].to_json, total_seconds: total_seconds, now: now) + end + end + + records + end + + def build_record(dimension:, bucket:, total_seconds:, now:, source_heartbeats_count: nil, source_max_heartbeat_time: nil, payload: nil) + { + user_id: @user.id, + dimension: dimension.to_s, + bucket_value: bucket.to_s, + bucket_value_present: !bucket.nil?, + total_seconds: total_seconds.to_i, + source_heartbeats_count: source_heartbeats_count, + source_max_heartbeat_time: source_max_heartbeat_time, + payload: payload, + created_at: now, + updated_at: now + } + end + + def rollup_eligible? + @params[:interval].blank? && @params[:from].blank? && @params[:to].blank? && FILTERS.none? { |field| @params[field].present? } + end + + def rollups_available? + DashboardRollup.table_exists? + rescue ActiveRecord::StatementInvalid + false + end + + def rollup_rows + return [] unless rollups_available? + + @rollup_rows ||= DashboardRollup.where(user_id: @user.id).to_a + end + + def rollup_rows_by_dimension + @rollup_rows_by_dimension ||= rollup_rows.group_by(&:dimension) + end + + def rollup_fragment_row(dimension) + rollup_rows_by_dimension.fetch(dimension.to_s, []).first + end + + def rollup_total_row + @rollup_total_row ||= rollup_rows.find(&:total_dimension?) + end + + def aggregate_rollup_stale?(total_row) + DashboardRollup.dirty?(@user.id) || time_fingerprint(total_row.source_max_heartbeat_time) != time_fingerprint(@user.heartbeats.maximum(:time)) + end + + def activity_graph_rollup_valid?(row) + payload = row&.payload + return false unless payload.is_a?(Hash) + + start_date, end_date = activity_graph_date_range(@user.timezone) + payload["timezone"] == @user.timezone && + payload["start_date"] == start_date && + payload["end_date"] == end_date && + payload["duration_by_date"].is_a?(Hash) + end + + def today_stats_rollup_valid?(row) + payload = row&.payload + return false unless payload.is_a?(Hash) + + payload["timezone"] == @user.timezone && + payload["today_date"] == today_date && + payload.key?("todays_duration_seconds") && + payload["todays_language_categories"].is_a?(Array) && + payload["todays_editor_keys"].is_a?(Array) + end + + def schedule_rollup_refresh(wait:) + return if @rollup_refresh_scheduled + + DashboardRollupRefreshJob.schedule_for(@user.id, wait: wait) + @rollup_refresh_scheduled = true + end + + def rollup_weekly_project_stats(rows) + result = week_ranges.to_h { |week_key, *_| [ week_key, {} ] } + rows.each do |row| + week_key, project = JSON.parse(row.bucket_value) + next unless result.key?(week_key) + + result[week_key][project] = row.total_seconds + end + result + end + + def time_fingerprint(timestamp) + return if timestamp.nil? + + (timestamp * 1_000_000).round + end + + def today_date + Time.use_zone(@user.timezone) { Date.current.iso8601 } + end + + def activity_graph_date_range(timezone) + Time.use_zone(timezone) { [ 365.days.ago.to_date.iso8601, Date.current.iso8601 ] } + end + + def week_ranges + Time.use_zone(@user.timezone) do + (0..11).map do |week_offset| + week_start = week_offset.weeks.ago.beginning_of_week + [ week_start.to_date.iso8601, week_start.to_f, week_offset.weeks.ago.end_of_week.to_f ] + end + end + end +end diff --git a/test/controllers/concerns/dashboard_data_test.rb b/test/controllers/concerns/dashboard_data_test.rb deleted file mode 100644 index ec49c61e7..000000000 --- a/test/controllers/concerns/dashboard_data_test.rb +++ /dev/null @@ -1,483 +0,0 @@ -require "test_helper" - -class DashboardDataTest < ActiveSupport::TestCase - include ActiveJob::TestHelper - - class Harness - include DashboardData - - attr_accessor :current_user, :params - end - - setup do - clear_enqueued_jobs - @original_queue_adapter = ActiveJob::Base.queue_adapter - ActiveJob::Base.queue_adapter = :test - end - - teardown do - clear_enqueued_jobs - ActiveJob::Base.queue_adapter = @original_queue_adapter - end - - test "raw filter options are cached per user" do - with_memory_cache_store do - Rails.cache.clear - - user = User.create!(timezone: "UTC") - harness = Harness.new - harness.current_user = user - harness.params = ActionController::Parameters.new - - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - - first = harness.send(:dashboard_live_raw_filter_options) - - create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "browsing") - - second = harness.send(:dashboard_live_raw_filter_options) - - assert_equal [ "alpha" ], first.fetch(:project) - assert_equal [ "alpha" ], second.fetch(:project) - assert_equal [ "ruby" ], second.fetch(:language) - end - end - - test "project grouped durations preserve nil project values" do - user = User.create!(timezone: "UTC") - harness = Harness.new - harness.current_user = user - harness.params = ActionController::Parameters.new - - Heartbeat.create!( - user: user, - time: Time.current.to_f - 60, - project: nil, - language: "ruby", - editor: "vscode", - operating_system: "macos", - category: "coding", - source_type: :test_entry - ) - Heartbeat.create!( - user: user, - time: Time.current.to_f, - project: nil, - language: "ruby", - editor: "vscode", - operating_system: "macos", - category: "coding", - source_type: :test_entry - ) - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - - scope = user.heartbeats - - assert_equal scope.group(:project).duration_seconds, harness.send(:dashboard_project_grouped_durations, scope) - end - - test "all-time dashboard data can be served from rollups" do - with_memory_cache_store do - Rails.cache.clear - - user = User.create!(timezone: "UTC") - harness = Harness.new - harness.current_user = user - harness.params = ActionController::Parameters.new - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") - end - - DashboardRollupRefreshService.new(user: user).call - - def harness.dashboard_grouped_durations_snapshot(_scope) - raise "expected rollup-backed dashboard path" - end - - def harness.dashboard_live_raw_filter_options - raise "expected rollup-backed filter options path" - end - - result = harness.send(:filterable_dashboard_data) - - assert_equal user.heartbeats.duration_seconds, result[:total_time] - assert_equal user.heartbeats.count, result[:total_heartbeats] - assert_equal "alpha", result["top_project"] - assert_equal [ "alpha", "beta" ], result[:project] - end - end - - test "all-time dashboard data falls back when rollup table is unavailable" do - with_memory_cache_store do - Rails.cache.clear - - user = User.create!(timezone: "UTC") - harness = Harness.new - harness.current_user = user - harness.params = ActionController::Parameters.new - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - end - - def harness.dashboard_rollups_available? - false - end - - result = harness.send(:filterable_dashboard_data) - - assert_equal user.heartbeats.duration_seconds, result[:total_time] - assert_equal "alpha", result["top_project"] - end - end - - test "homepage rollup path falls back to live filter options when filter option rollup is missing" do - with_memory_cache_store do - Rails.cache.clear - - user = User.create!(timezone: "UTC") - harness = build_harness(user) - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - end - - DashboardRollupRefreshService.new(user: user).call - DashboardRollup.find_by!(user: user, dimension: DashboardRollup::FILTER_OPTIONS_DIMENSION).destroy! - - clear_enqueued_jobs - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - result = nil - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - result = harness.send(:filterable_dashboard_data) - end - - assert_equal [ "alpha" ], result[:project] - assert_equal [ "Ruby" ], result[:language] - end - end - - test "dirty rollup serves last rollup and schedules a refresh" do - with_memory_cache_store do - Rails.cache.clear - - user = User.create!(timezone: "UTC") - harness = Harness.new - harness.current_user = user - harness.params = ActionController::Parameters.new - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - end - - DashboardRollupRefreshService.new(user: user).call - - def harness.dashboard_grouped_durations_snapshot(_scope) - raise "expected rollup-backed dashboard path" - end - - total_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TOTAL_DIMENSION) - - clear_enqueued_jobs - travel 1.minute do - create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") - end - - result = nil - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) - assert_no_enqueued_jobs(only: DashboardRollupRefreshJob) do - result = harness.send(:filterable_dashboard_data) - end - - assert_equal total_row.total_seconds, result[:total_time] - assert_equal total_row.source_heartbeats_count, result[:total_heartbeats] - assert_equal "alpha", result["top_project"] - assert_equal [ "alpha" ], result[:project] - end - end - - test "stale rollup fingerprint serves last rollup and schedules a refresh" do - with_memory_cache_store do - Rails.cache.clear - - user = User.create!(timezone: "UTC") - harness = Harness.new - harness.current_user = user - harness.params = ActionController::Parameters.new - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - travel 1.minute - create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - end - - DashboardRollupRefreshService.new(user: user).call - - def harness.dashboard_grouped_durations_snapshot(_scope) - raise "expected rollup-backed dashboard path" - end - - total_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TOTAL_DIMENSION) - - travel 1.minute do - create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") - end - - DashboardRollup.clear_dirty(user.id) - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - result = nil - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - result = harness.send(:filterable_dashboard_data) - end - - assert_equal total_row.total_seconds, result[:total_time] - assert_equal total_row.source_heartbeats_count, result[:total_heartbeats] - assert_equal "alpha", result["top_project"] - assert_equal [ "alpha" ], result[:project] - end - end - - test "today stats and activity graph can be served from rollups" do - with_memory_cache_store do - Rails.cache.clear - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - user = User.create!(timezone: "UTC") - harness = build_harness(user) - - create_heartbeat_at(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - - DashboardRollupRefreshService.new(user: user).call - - def harness.dashboard_live_today_stats_data - raise "expected rollup-backed today stats path" - end - - def harness.dashboard_live_activity_graph_data - raise "expected rollup-backed activity graph path" - end - - today_stats = harness.send(:today_stats_data) - activity_graph = harness.send(:activity_graph_data) - - assert today_stats[:show_logged_time_sentence] - assert_equal [ ApplicationController.helpers.display_language_name("ruby") ], today_stats[:todays_languages] - assert_equal [ ApplicationController.helpers.display_editor_name("vscode") ], today_stats[:todays_editors] - assert_equal ApplicationController.helpers.short_time_detailed(120), today_stats[:todays_duration_display] - assert_equal "2025-04-14", activity_graph[:start_date] - assert_equal "2026-04-14", activity_graph[:end_date] - assert_equal 120, activity_graph[:duration_by_date]["2026-04-14"] - end - end - end - - test "invalid today stats rollup recalculates only today stats and schedules a refresh" do - with_memory_cache_store do - Rails.cache.clear - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - user = User.create!(timezone: "UTC") - harness = build_harness(user) - - create_heartbeat_at(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - - DashboardRollupRefreshService.new(user: user).call - - today_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TODAY_STATS_DIMENSION) - today_row.update!(payload: today_row.payload.merge("today_date" => "2026-04-13")) - - def harness.dashboard_grouped_durations_snapshot(_scope) - raise "expected rollup-backed dashboard path" - end - - def harness.dashboard_live_today_stats_data - { source: :live_today } - end - - def harness.dashboard_live_activity_graph_data - raise "expected rollup-backed activity graph path" - end - - clear_enqueued_jobs - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - aggregate = nil - today_stats = nil - activity_graph = nil - - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - aggregate = harness.send(:filterable_dashboard_data) - today_stats = harness.send(:today_stats_data) - activity_graph = harness.send(:activity_graph_data) - end - - assert_equal 120, aggregate[:total_time] - assert_equal({ source: :live_today }, today_stats) - assert_equal 120, activity_graph[:duration_by_date]["2026-04-14"] - assert_equal 1, enqueued_jobs.count { |job| job[:job] == DashboardRollupRefreshJob } - end - end - end - - test "invalid activity graph rollup recalculates only activity graph and schedules a refresh" do - with_memory_cache_store do - Rails.cache.clear - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - user = User.create!(timezone: "UTC") - harness = build_harness(user) - - create_heartbeat_at(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - - DashboardRollupRefreshService.new(user: user).call - - activity_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::ACTIVITY_GRAPH_DIMENSION) - activity_row.update!(payload: activity_row.payload.merge("end_date" => "2026-04-13")) - - def harness.dashboard_grouped_durations_snapshot(_scope) - raise "expected rollup-backed dashboard path" - end - - def harness.dashboard_live_today_stats_data - raise "expected rollup-backed today stats path" - end - - def harness.dashboard_live_activity_graph_data - { source: :live_activity } - end - - clear_enqueued_jobs - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - aggregate = nil - today_stats = nil - activity_graph = nil - - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - aggregate = harness.send(:filterable_dashboard_data) - today_stats = harness.send(:today_stats_data) - activity_graph = harness.send(:activity_graph_data) - end - - assert_equal 120, aggregate[:total_time] - assert today_stats[:show_logged_time_sentence] - assert_equal({ source: :live_activity }, activity_graph) - assert_equal 1, enqueued_jobs.count { |job| job[:job] == DashboardRollupRefreshJob } - end - end - end - - test "missing today stats and activity graph rollups recalculate only those fragments and schedule one refresh" do - with_memory_cache_store do - Rails.cache.clear - - travel_to Time.utc(2026, 4, 14, 12, 0, 0) do - user = User.create!(timezone: "UTC") - harness = build_harness(user) - - create_heartbeat_at(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - - DashboardRollupRefreshService.new(user: user).call - DashboardRollup.where( - user: user, - dimension: [ DashboardRollup::TODAY_STATS_DIMENSION, DashboardRollup::ACTIVITY_GRAPH_DIMENSION ] - ).delete_all - - def harness.dashboard_grouped_durations_snapshot(_scope) - raise "expected rollup-backed dashboard path" - end - - def harness.dashboard_live_today_stats_data - { source: :live_today } - end - - def harness.dashboard_live_activity_graph_data - { source: :live_activity } - end - - clear_enqueued_jobs - Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) - - aggregate = nil - today_stats = nil - activity_graph = nil - - assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do - aggregate = harness.send(:filterable_dashboard_data) - today_stats = harness.send(:today_stats_data) - activity_graph = harness.send(:activity_graph_data) - end - - assert_equal 120, aggregate[:total_time] - assert_equal({ source: :live_today }, today_stats) - assert_equal({ source: :live_activity }, activity_graph) - assert_equal 1, enqueued_jobs.count { |job| job[:job] == DashboardRollupRefreshJob } - end - end - end - - private - - def build_harness(user, params: {}) - harness = Harness.new - harness.current_user = user - harness.params = ActionController::Parameters.new(params) - harness - end - - def with_memory_cache_store - original_cache = Rails.cache - Rails.cache = ActiveSupport::Cache.lookup_store(:memory_store) - yield - ensure - Rails.cache = original_cache - end - - def create_heartbeat(user, project:, language:, editor:, operating_system:, category:) - Heartbeat.create!( - user: user, - time: Time.current.to_f, - project: project, - language: language, - editor: editor, - operating_system: operating_system, - category: category, - source_type: :test_entry - ) - end - - def create_heartbeat_at(user, timestamp, project:, language:, editor:, operating_system:, category:) - Heartbeat.create!( - user: user, - time: Time.parse(timestamp).to_f, - project: project, - language: language, - editor: editor, - operating_system: operating_system, - category: category, - source_type: :test_entry - ) - end -end diff --git a/test/controllers/static_pages_controller_test.rb b/test/controllers/static_pages_controller_test.rb index 2af28bbdd..6d4f241f1 100644 --- a/test/controllers/static_pages_controller_test.rb +++ b/test/controllers/static_pages_controller_test.rb @@ -11,7 +11,7 @@ class StaticPagesControllerTest < ActionDispatch::IntegrationTest create_heartbeat(user, "2026-04-13 10:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") create_heartbeat(user, "2026-04-13 10:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - DashboardRollupRefreshService.new(user: user).call + DashboardSnapshot.new(user: user).persist_rollups! get root_path diff --git a/test/services/dashboard_rollup_refresh_service_test.rb b/test/services/dashboard_snapshot_rollup_persistence_test.rb similarity index 96% rename from test/services/dashboard_rollup_refresh_service_test.rb rename to test/services/dashboard_snapshot_rollup_persistence_test.rb index 926294ed6..889518316 100644 --- a/test/services/dashboard_rollup_refresh_service_test.rb +++ b/test/services/dashboard_snapshot_rollup_persistence_test.rb @@ -1,6 +1,6 @@ require "test_helper" -class DashboardRollupRefreshServiceTest < ActiveSupport::TestCase +class DashboardSnapshotRollupPersistenceTest < ActiveSupport::TestCase test "rebuilds dashboard rollups from current heartbeat aggregates" do travel_to Time.utc(2026, 4, 14, 12, 0, 0) do user = User.create!(timezone: "UTC") @@ -14,7 +14,7 @@ class DashboardRollupRefreshServiceTest < ActiveSupport::TestCase create_heartbeat(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") create_heartbeat(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") - DashboardRollupRefreshService.new(user: user).call + DashboardSnapshot.new(user: user).persist_rollups! total_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TOTAL_DIMENSION) filter_options_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::FILTER_OPTIONS_DIMENSION) diff --git a/test/services/dashboard_snapshot_test.rb b/test/services/dashboard_snapshot_test.rb new file mode 100644 index 000000000..b3817c005 --- /dev/null +++ b/test/services/dashboard_snapshot_test.rb @@ -0,0 +1,170 @@ +require "test_helper" + +class DashboardSnapshotTest < ActiveSupport::TestCase + include ActiveJob::TestHelper + + setup do + clear_enqueued_jobs + @original_queue_adapter = ActiveJob::Base.queue_adapter + ActiveJob::Base.queue_adapter = :test + end + + teardown do + clear_enqueued_jobs + ActiveJob::Base.queue_adapter = @original_queue_adapter + end + + test "raw filter options are cached per user" do + with_memory_cache_store do + Rails.cache.clear + user = User.create!(timezone: "UTC") + + create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + first = DashboardSnapshot.new(user: user, params: { interval: "week" }).call + + create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "browsing") + second = DashboardSnapshot.new(user: user, params: { interval: "week" }).call + + assert_equal [ "alpha" ], first.filterable_dashboard_data.fetch(:project) + assert_equal [ "alpha" ], second.filterable_dashboard_data.fetch(:project) + assert_equal [ "Ruby" ], second.filterable_dashboard_data.fetch(:language) + end + end + + test "all-time dashboard data can be served from rollups with source metadata" do + with_memory_cache_store do + Rails.cache.clear + user = User.create!(timezone: "UTC") + + travel_to Time.utc(2026, 4, 14, 12, 0, 0) do + create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + travel 1.minute + create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + travel 1.minute + create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") + end + + DashboardSnapshot.new(user: user).persist_rollups! + result = DashboardSnapshot.new(user: user).call + + assert_equal user.heartbeats.duration_seconds, result.filterable_dashboard_data[:total_time] + assert_equal user.heartbeats.count, result.filterable_dashboard_data[:total_heartbeats] + assert_equal "alpha", result.filterable_dashboard_data["top_project"] + assert_equal [ "alpha", "beta" ], result.filterable_dashboard_data[:project] + assert_equal :rollup, result.sources[:aggregate] + assert_equal :rollup, result.sources[:filter_options] + end + end + + test "dirty rollup serves last rollup and schedules a refresh" do + with_memory_cache_store do + Rails.cache.clear + user = User.create!(timezone: "UTC") + + travel_to Time.utc(2026, 4, 14, 12, 0, 0) do + create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + travel 1.minute + create_heartbeat(user, project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + end + + DashboardSnapshot.new(user: user).persist_rollups! + total_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TOTAL_DIMENSION) + + clear_enqueued_jobs + Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) + travel 1.minute do + create_heartbeat(user, project: "beta", language: "javascript", editor: "zed", operating_system: "linux", category: "coding") + end + clear_enqueued_jobs + Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) + + result = nil + assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do + result = DashboardSnapshot.new(user: user).call + end + + assert_equal total_row.total_seconds, result.filterable_dashboard_data[:total_time] + assert_equal total_row.source_heartbeats_count, result.filterable_dashboard_data[:total_heartbeats] + assert_equal "alpha", result.filterable_dashboard_data["top_project"] + assert_equal :stale_rollup, result.sources[:aggregate] + assert_equal 1, enqueued_jobs.count { |job| job[:job] == DashboardRollupRefreshJob } + end + end + + test "today stats and activity graph can be served from rollups" do + with_memory_cache_store do + Rails.cache.clear + + travel_to Time.utc(2026, 4, 14, 12, 0, 0) do + user = User.create!(timezone: "UTC") + create_heartbeat_at(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + + DashboardSnapshot.new(user: user).persist_rollups! + result = DashboardSnapshot.new(user: user).call + + assert result.today_stats[:show_logged_time_sentence] + assert_equal [ ApplicationController.helpers.display_language_name("ruby") ], result.today_stats[:todays_languages] + assert_equal [ ApplicationController.helpers.display_editor_name("vscode") ], result.today_stats[:todays_editors] + assert_equal ApplicationController.helpers.short_time_detailed(120), result.today_stats[:todays_duration_display] + assert_equal "2025-04-14", result.activity_graph[:start_date] + assert_equal "2026-04-14", result.activity_graph[:end_date] + assert_equal 120, result.activity_graph[:duration_by_date]["2026-04-14"] + assert_equal :rollup, result.sources[:today_stats] + assert_equal :rollup, result.sources[:activity_graph] + end + end + end + + test "invalid fragments recalculate only invalid fragments and schedule one refresh" do + with_memory_cache_store do + Rails.cache.clear + + travel_to Time.utc(2026, 4, 14, 12, 0, 0) do + user = User.create!(timezone: "UTC") + create_heartbeat_at(user, "2026-04-14 09:00:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + create_heartbeat_at(user, "2026-04-14 09:01:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + create_heartbeat_at(user, "2026-04-14 09:02:00 UTC", project: "alpha", language: "ruby", editor: "vscode", operating_system: "macos", category: "coding") + + DashboardSnapshot.new(user: user).persist_rollups! + today_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::TODAY_STATS_DIMENSION) + today_row.update!(payload: today_row.payload.merge("today_date" => "2026-04-13")) + activity_row = DashboardRollup.find_by!(user: user, dimension: DashboardRollup::ACTIVITY_GRAPH_DIMENSION) + activity_row.update!(payload: activity_row.payload.merge("end_date" => "2026-04-13")) + + clear_enqueued_jobs + Rails.cache.delete(DashboardRollupRefreshJob.enqueue_cache_key(user.id)) + + result = nil + assert_enqueued_with(job: DashboardRollupRefreshJob, args: [ user.id ]) do + result = DashboardSnapshot.new(user: user).call + end + + assert_equal 120, result.filterable_dashboard_data[:total_time] + assert_equal :rollup, result.sources[:aggregate] + assert_equal :live, result.sources[:today_stats] + assert_equal :live, result.sources[:activity_graph] + assert_equal 1, enqueued_jobs.count { |job| job[:job] == DashboardRollupRefreshJob } + end + end + end + + private + + def with_memory_cache_store + original_cache = Rails.cache + Rails.cache = ActiveSupport::Cache.lookup_store(:memory_store) + yield + ensure + Rails.cache = original_cache + end + + def create_heartbeat(user, project:, language:, editor:, operating_system:, category:) + Heartbeat.create!(user: user, time: Time.current.to_f, project: project, language: language, editor: editor, operating_system: operating_system, category: category, source_type: :test_entry) + end + + def create_heartbeat_at(user, timestamp, project:, language:, editor:, operating_system:, category:) + Heartbeat.create!(user: user, time: Time.parse(timestamp).to_f, project: project, language: language, editor: editor, operating_system: operating_system, category: category, source_type: :test_entry) + end +end