diff --git a/development/migrations/2026_05_05_backfill_missing_order.clj b/development/migrations/2026_05_05_backfill_missing_order.clj new file mode 100644 index 000000000..fa52f7ddc --- /dev/null +++ b/development/migrations/2026_05_05_backfill_missing_order.clj @@ -0,0 +1,184 @@ +(ns migrations.2026-05-05-backfill-missing-order + (:require [behave-cms.server :as cms] + [behave-cms.store :refer [default-conn]] + [datomic.api :as d] + [schema-migrate.interface :as sm])) + +;; =========================================================================================================== +;; Overview +;; =========================================================================================================== + +;; Normalize /order values across all sibling sets: for every parent that +;; has a non-dense ordering of its children (missing values, duplicates, or +;; gaps), renumber the children 0..N-1 while preserving their current +;; relative order. Ties (duplicate orders) are broken by :db/id, and +;; children currently missing /order are appended after those that have +;; one (also :db/id-sorted). +;; +;; Only children whose order actually changes are emitted in the payload. +;; +;; Some list-options are shared across multiple parent lists. Shared options +;; that would receive the same order value in every parent are deduped safely. +;; Shared options with conflicting computed orders (different values under +;; different parents) are excluded from the payload entirely and reported via +;; the inspection comment block below. Known conflict at time of authoring: +;; eid 4611681620380878252 (fuelmodeltype:s) — shared by FuelModelType and +;; CompassDirection, which is itself a likely data bug to investigate +;; separately. +;; +;; Counts at time of authoring (~503 total emitted ops): +;; :list-option/order — ~346 (after dedupe/skip) +;; :group-variable/order — 69 +;; :group/order — 41 +;; :submodule/order — 23 +;; :subtool-variable/order — 17 +;; :tool/order — 3 +;; :module/order — 2 +;; :subtool/order — 2 +;; Of these, 282 are entities that previously had no /order at all. + +;; =========================================================================================================== +;; Initialize +;; =========================================================================================================== + +(cms/init-db!) + +#_{:clj-kondo/ignore [:missing-docstring]} +(def conn (default-conn)) + +;; =========================================================================================================== +;; Payload +;; =========================================================================================================== + +#_{:clj-kondo/ignore [:missing-docstring]} +(def order-pairs + [[:module/order :application/modules] + [:submodule/order :module/submodules] + [:group/order :submodule/groups] + [:group-variable/order :group/group-variables] + [:tool/order :application/tools] + [:subtool/order :tool/subtools] + [:subtool-variable/order :subtool/variables] + [:list-option/order :list/options] + [:note-category/order :application/note-categories] + [:search-table/order :module/search-tables] + [:search-table-column/order :search-table/columns] + [:tag/order :tag-set/tags] + [:pivot-column/order :pivot-table/columns]]) + +#_{:clj-kondo/ignore [:missing-docstring]} +(defn build-renumber-payload + [db parent-ref order-attr] + (let [parent-ids (d/q [:find '[?p ...] :where ['?p parent-ref]] db)] + (mapcat + (fn [pid] + (let [children (->> (parent-ref (d/entity db pid)) (map :db/id)) + order-of (fn [eid] (order-attr (d/entity db eid))) + orders (map order-of children) + needs-fix? (or (some nil? orders) + (not= (sort orders) (range (count children)))) + sorted-eids (sort-by (juxt #(or (order-of %) Long/MAX_VALUE) identity) + children)] + (when needs-fix? + (keep-indexed + (fn [i eid] + (when (not= i (order-of eid)) + {:db/id eid order-attr i})) + sorted-eids)))) + parent-ids))) + +#_{:clj-kondo/ignore [:missing-docstring]} +(def raw-payload + (let [db (d/db conn)] + (vec (mapcat (fn [[oa parent-ref]] (build-renumber-payload db parent-ref oa)) order-pairs)))) + +#_{:clj-kondo/ignore [:missing-docstring]} +(def conflicting-eids + (->> raw-payload + (group-by :db/id) + (filter (fn [[_ entries]] + (> (count (distinct (map #(dissoc % :db/id) entries))) 1))) + (map first) + set)) + +#_{:clj-kondo/ignore [:missing-docstring]} +(def payload + (->> raw-payload + (remove #(contains? conflicting-eids (:db/id %))) + distinct + vec)) + +;; Inspect conflicting eids before transacting (should be #{4611681620380878252}) +(comment + (let [db (d/db conn)] + (for [eid conflicting-eids] + {:eid eid + :parents (d/q '[:find ?p ?n + :in $ ?c + :where [?p :list/options ?c] + [(get-else $ ?p :list/name "") ?n]] + db eid)}))) + +;; =========================================================================================================== +;; Data fix — CompassDirection erroneously references the FuelModelType "S" option +;; =========================================================================================================== + +;; The list-option eid 4611681620380878252 ("S", value "8", translation-key +;; "behave:list-option:list-option:fuelmodeltype:s") belongs to FuelModelType +;; but is also referenced by CompassDirection. Create a new dedicated +;; CompassDirection:S option with the correct translation-key prefix and +;; remove the erroneous reference. The new option is given order 8 — its +;; 0-indexed alphabetical slot between SSE (7 post-renumber) and SSW (9). The +;; renumber pass above resolves any transient duplicates (CompassDirection's +;; existing orders are 1..16) by sorting on (order, :db/id) and producing a +;; final 0..N-1 sequence. +;; +;; Transact this payload BEFORE the renumber payload above so the renumber +;; pass can safely include the (now non-shared) FuelModelType:S option. + +#_{:clj-kondo/ignore [:missing-docstring]} +(def compass-direction-eid 4611681620380878243) + +#_{:clj-kondo/ignore [:missing-docstring]} +(def fuelmodeltype-s-eid 4611681620380878252) + +#_{:clj-kondo/ignore [:missing-docstring]} +(def data-fix-payload + [{:db/id "compass-direction-s" + :bp/uuid (str (java.util.UUID/randomUUID)) + :list-option/name "S" + :list-option/value "8" + :list-option/order 8 + :list-option/translation-key "behave:list-option:list-option:compassdirection:s"} + [:db/add compass-direction-eid :list/options "compass-direction-s"] + [:db/retract compass-direction-eid :list/options fuelmodeltype-s-eid]]) + +#_{:clj-kondo/ignore [:missing-docstring]} +(def add-new-translations-payload + (sm/build-translations-payload conn 100 + {"behave:list-option:list-option:compassdirection:s" "S"})) + +;; =========================================================================================================== +;; Transact Payload +;; =========================================================================================================== + +;; Step 1 — fix the erroneous CompassDirection -> fuelmodeltype:s reference. +(comment + #_{:clj-kondo/ignore [:missing-docstring]} + (try (def data-fix-tx-data @(d/transact conn (concat data-fix-payload add-new-translations-payload))) + (catch Exception e (str "caught exception: " (.getMessage e))))) + +;; Step 2 — re-evaluate `raw-payload`/`conflicting-eids`/`payload` (above) so +;; the renumber pass picks up the cleaned-up DB state, then transact. +(comment + #_{:clj-kondo/ignore [:missing-docstring]} + (try (def tx-data @(d/transact conn payload)) + (catch Exception e (str "caught exception: " (.getMessage e))))) + +;; =========================================================================================================== +;; In case we need to rollback. +;; =========================================================================================================== + +(comment + (sm/rollback-tx! conn tx-data) + (sm/rollback-tx! conn data-fix-tx-data)) diff --git a/projects/behave_cms/src/cljs/behave_cms/components/search_table.cljs b/projects/behave_cms/src/cljs/behave_cms/components/search_table.cljs index c221e60a8..dddd01056 100644 --- a/projects/behave_cms/src/cljs/behave_cms/components/search_table.cljs +++ b/projects/behave_cms/src/cljs/behave_cms/components/search_table.cljs @@ -1,11 +1,11 @@ (ns behave-cms.components.search-table - (:require [behave-cms.components.conditionals.views :refer [conditionals-graph manage-conditionals]] - [behave-cms.components.common :refer [accordion btn-sm]] - [behave-cms.components.entity-form :refer [entity-form]] - [behave-cms.components.table-entity-form :refer [table-entity-form]] + (:require [behave-cms.components.common :refer [accordion btn-sm]] + [behave-cms.components.conditionals.views :refer [conditionals-graph manage-conditionals]] + [behave-cms.components.entity-form :refer [entity-form]] + [behave-cms.components.table-entity-form :refer [table-entity-form]] [behave-cms.submodules.subs] - [reagent.core :as r] - [re-frame.core :as rf])) + [re-frame.core :as rf] + [reagent.core :as r])) (defn search-tables "Component for displaying all existing search tables and a form for entering new ones" @@ -102,7 +102,9 @@ :outline-danger "Delete Search Table" #(when (js/confirm (str "Are you sure you want to delete this search table?")) - (rf/dispatch [:api/delete-entity search-table-id]))]]]] + (rf/dispatch [:api/delete-entity search-table-id + {:order-attr :search-table/order + :siblings (:module/search-tables @module)}]))]]]] [:hr]])) (:module/search-tables @module))) (if @show-add-search-table? @@ -123,7 +125,12 @@ :type :radio :options [{:label "Minimum" :value :min} {:label "Maximum" :value :max}]}] - :on-create #(do (swap! show-add-search-table? not) %)}] + :on-create #(let [siblings (:module/search-tables @module)] + (swap! show-add-search-table? not) + (assoc % :search-table/order + (if (seq siblings) + (inc (apply max (keep :search-table/order siblings))) + 0)))}] [btn-sm :primary "Add Search Table" diff --git a/projects/behave_cms/src/cljs/behave_cms/components/table_entity_form.cljs b/projects/behave_cms/src/cljs/behave_cms/components/table_entity_form.cljs index f33042027..ba39df20a 100644 --- a/projects/behave_cms/src/cljs/behave_cms/components/table_entity_form.cljs +++ b/projects/behave_cms/src/cljs/behave_cms/components/table_entity_form.cljs @@ -87,7 +87,8 @@ :on-delete (when modify? #(when (js/confirm (str "Are you sure you want to delete this " (name entity))) - (rf/dispatch-sync [:api/delete-entity (:db/id %)]))) + (rf/dispatch-sync [:api/delete-entity (:db/id %) + (when order-attr {:order-attr order-attr :siblings entities})]))) :on-select (when modify? #(if (and @show-entity-form? (= @entity-id-atom (:db/id %))) (do (reset! entity-id-atom nil) @@ -119,7 +120,9 @@ (swap! show-entity-form? not) (cond-> % translation-config (create-translation! entity translation-config) - order-attr (assoc order-attr (count entities))))}] + order-attr (assoc order-attr (if (seq entities) + (inc (apply max (keep order-attr entities))) + 0))))}] (when (and (seq translation-attrs) @entity-id-atom) (let [entity-data @(rf/subscribe [:re-entity @entity-id-atom])] [:<> diff --git a/projects/behave_cms/src/cljs/behave_cms/events.cljs b/projects/behave_cms/src/cljs/behave_cms/events.cljs index 900df71b7..220ac3129 100644 --- a/projects/behave_cms/src/cljs/behave_cms/events.cljs +++ b/projects/behave_cms/src/cljs/behave_cms/events.cljs @@ -7,13 +7,11 @@ [behave-cms.languages.events] [behave-cms.lists.events] [behave-cms.modules.events] - [behave-cms.routes :refer [app-routes singular]] [behave-cms.subgroups.events] [behave-cms.submodules.events] [behave-cms.subtools.events] [behave-cms.utils :as u] [behave-cms.variables.events] - [bidi.bidi :refer [path-for]] [clojure.core.async :refer [go (+ curr-position 1) @@ -98,7 +100,7 @@ (reg-event-fx :refresh - (fn [{db :db} [_ new-route]] + (fn [_ [_ new-route]] (set! (.-location js/window) new-route))) (reg-event-db @@ -113,64 +115,66 @@ (reg-event-db :state/set-state (path :state) - (fn [db [_ path value]] + (fn [db [_ p value]] (cond - (or (vector? path) (list? path)) - (assoc-in db path value) + (or (vector? p) (list? p)) + (assoc-in db p value) - (keyword? path) - (assoc db path value) + (keyword? p) + (assoc db p value) :else db))) (reg-event-db :state/update (path :state) - (fn [db [_ path f]] + (fn [db [_ p f]] (cond - (or (vector? path) (list? path)) - (update-in db path f) + (or (vector? p) (list? p)) + (update-in db p f) - (keyword? path) - (update db path f) + (keyword? p) + (update db p f) :else db))) (reg-event-db :state/select (path [:state :selected]) - (fn [db [_ path value]] - (if (keyword? path) - (assoc db path value) + (fn [db [_ p value]] + (if (keyword? p) + (assoc db p value) db))) (reg-event-db :state/merge (path :state) - (fn [state [_ path value]] - (let [orig-value (get-in state path)] + (fn [state [_ p value]] + (let [orig-value (get-in state p)] (cond (nil? orig-value) - (assoc-in state path value) + (assoc-in state p value) (and (map? orig-value) (map? value)) - (update-in state path merge value) + (update-in state p merge value) (and (or (vector? orig-value) (seq? orig-value)) (or (vector? value) (seq? orig-value))) - (update-in state path into value))))) + (update-in state p into value))))) (reg-event-db :state/remove-nth (path :state) - (fn [db [_ path n]] - (let [v (get-in db path)] - (println [:REMOVE-NTH [:ARGS path n] path v (remove-nth v n) (assoc-in db path (remove-nth v n))]) - (assoc-in db path (remove-nth v n))))) + (fn [db [_ p n]] + (let [v (get-in db p)] + (println [:REMOVE-NTH [:ARGS p n] p v (remove-nth v n) (assoc-in db p (remove-nth v n))]) + (assoc-in db p (remove-nth v n))))) ;;; AJAX/Fetch Effects -(defn request [{:keys [uri method data on-success on-error fn-args]}] +(defn request + "Issue an EDN ajax request and dispatch on success/error." + [{:keys [uri method data on-success on-error fn-args]}] (let [handler (fn [[ok result]] (dispatch [(if ok on-success on-error) result @@ -184,7 +188,9 @@ (reg-fx :api/request request) -(defn http-request [{:keys [uri method body on-success on-error fn-args]}] +(defn http-request + "Issue an HTTP request via core.async and dispatch on success/error." + [{:keys [uri method body on-success on-error fn-args]}] (go (let [response (> siblings + (remove #(= deleted-id (:db/id %))) + (sort-by order-attr) + (keep-indexed + (fn [i e] + (when (not= i (get e order-attr)) + {:db/id (:db/id e) order-attr i})))))] + {:transact (into [[:db.fn/retractEntity id]] renumber)}))) (reg-event-fx :api/reorder diff --git a/projects/behave_cms/src/cljs/behave_cms/subgroups/views.cljs b/projects/behave_cms/src/cljs/behave_cms/subgroups/views.cljs index 1b780f361..854150a85 100644 --- a/projects/behave_cms/src/cljs/behave_cms/subgroups/views.cljs +++ b/projects/behave_cms/src/cljs/behave_cms/subgroups/views.cljs @@ -68,8 +68,8 @@ :group-variable/translation-key (str @translation-key ":" (->kebab (:variable/name variable))) :group-variable/result-translation-key (-> (str/replace @translation-key #":input:|:output:" ":result:") (str ":" (->kebab (:variable/name variable)))) - :group-variable/help-key (str @translation-key ":" (->kebab (:variable/name variable)) ":help") - :group-variable/order (count @group-variables)}])) + :group-variable/help-key (str @translation-key ":" (->kebab (:variable/name variable)) ":help")} + {:order-attr :group-variable/order :siblings @group-variables}])) :on-blur #(rf/dispatch [:state/set-state [:search :variables] nil])}]])) ;;; Settings diff --git a/projects/behave_cms/src/cljs/behave_cms/submodules/views.cljs b/projects/behave_cms/src/cljs/behave_cms/submodules/views.cljs index 726cb2b08..99ec9c31d 100644 --- a/projects/behave_cms/src/cljs/behave_cms/submodules/views.cljs +++ b/projects/behave_cms/src/cljs/behave_cms/submodules/views.cljs @@ -48,7 +48,7 @@ {:value "max" :label "max"} {:value "count" :label "count"}]}))}])) -(defn- submodules-table [module-id app-id] +(defn- submodules-table [module-id] (let [selected-state-path [:selected :submodule] editor-state-path [:editors :submodule] submodule (rf/subscribe [:submodules module-id])] @@ -57,8 +57,8 @@ :form-state-path editor-state-path :entities (sort-by :submodule/order @submodule) :on-select (table-entity-form-on-select selected-state-path) - :parent-id app-id - :parent-field :application/_submodules + :parent-id module-id + :parent-field :module/_submodules :table-header-attrs [:submodule/name :submodule/io] :order-attr :submodule/order :entity-form-fields [{:label "Name" @@ -70,16 +70,16 @@ :options [{:label "Input" :value :input} {:label "Output" :value :output}]}]}])) -(defn- submodules-results-order-table [module-id app-id] +(defn- submodules-results-order-table [module-id] (let [submodule (rf/subscribe [:submodules module-id])] [table-entity-form {:entity :submodule :entities (sort-by :submodule/results-order @submodule) :modify? false - :parent-id app-id - :parent-field :application/_submodules + :parent-id module-id + :parent-field :module/_submodules :table-header-attrs [:submodule/name :submodule/io] - :order-attr :submodule/order + :order-attr :submodule/results-order :entity-form-fields [{:label "Name" :required? true :field-key :submodule/name}]}])) @@ -106,11 +106,11 @@ [:h2 (:module/name @module)]] [accordion "Submodules" - [submodules-table (:db/id @module) (:db/id application)]] + [submodules-table (:db/id @module)]] [:hr] [accordion "Submodules Results Order" - [submodules-results-order-table (:db/id @module) (:db/id application)]] + [submodules-results-order-table (:db/id @module)]] [:hr] [accordion "Translations" diff --git a/projects/behave_cms/src/cljs/behave_cms/subtools/views.cljs b/projects/behave_cms/src/cljs/behave_cms/subtools/views.cljs index bc4b72b26..fa019b758 100644 --- a/projects/behave_cms/src/cljs/behave_cms/subtools/views.cljs +++ b/projects/behave_cms/src/cljs/behave_cms/subtools/views.cljs @@ -1,17 +1,17 @@ (ns behave-cms.subtools.views - (:require [clojure.set :refer [difference]] - [reagent.core :as r] - [re-frame.core :as rf] - [string-utils.interface :refer [->kebab]] - [behave-cms.components.common :refer [accordion radio-buttons simple-table window checkbox]] + (:require [behave-cms.components.common :refer [accordion radio-buttons simple-table window checkbox]] [behave-cms.components.cpp-editor :refer [cpp-editor-form]] - [behave-cms.help.views :refer [help-editor]] [behave-cms.components.sidebar :refer [->sidebar-links sidebar sidebar-width]] [behave-cms.components.translations :refer [all-translations]] [behave-cms.components.variable-search :refer [variable-search]] - [behave-cms.utils :as u] + [behave-cms.events] + [behave-cms.help.views :refer [help-editor]] [behave-cms.subs] - [behave-cms.events])) + [behave-cms.utils :as u] + [clojure.set :refer [difference]] + [re-frame.core :as rf] + [reagent.core :as r] + [string-utils.interface :refer [->kebab]])) ;;; Constants @@ -42,9 +42,9 @@ {:variable/_subtool-variables % :subtool-variable/translation-key (str translation-key ":" (->kebab (:variable/name variable))) :subtool-variable/help-key (str translation-key ":" (->kebab (:variable/name variable)) ":help") - :subtool-variable/order (count variables) :subtool-variable/io io - :subtool/_variables subtool-id}])) + :subtool/_variables subtool-id} + {:order-attr :subtool-variable/order :siblings variables}])) :on-blur #(rf/dispatch [:state/set-state [:search :variables] nil])}])) (defn- manage-variable [subtool-id translation-key variables] @@ -74,7 +74,8 @@ {:on-select #(rf/dispatch [:subtool/edit-variable (first (:variable/_subtool-variables %))]) :on-delete #(when (js/confirm (str "Are you sure you want to delete the variable " (:variable/name %) "?")) - (rf/dispatch [:api/delete-entity %])) + (rf/dispatch [:api/delete-entity % + {:order-attr :subtool-variable/order :siblings variables}])) :on-increase #(rf/dispatch [:api/reorder % variables :subtool-variable/order :inc]) :on-decrease #(rf/dispatch [:api/reorder % variables :subtool-variable/order :dec])}]])