diff --git a/mix.lock b/mix.lock index c17c7cba..95b7eab9 100644 --- a/mix.lock +++ b/mix.lock @@ -19,16 +19,14 @@ "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, "decimal": {:hex, :decimal, "2.2.0", "df3d06bb9517e302b1bd265c1e7f16cda51547ad9d99892049340841f3e15836", [:mix], [], "hexpm", "af8daf87384b51b7e611fb1a1f2c4d4876b65ef968fa8bd3adf44cff401c7f21"}, "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"}, - "delta_crdt": {:hex, :delta_crdt, "0.6.4", "79d235eef82a58bb0cb668bc5b9558d2e65325ccb46b74045f20b36fd41671da", [:mix], [{:merkle_map, "~> 0.2.0", [hex: :merkle_map, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4a81f579c06aeeb625db54c6c109859a38aa00d837e3e7f8ac27b40cea34885a"}, + "delta_crdt": {:hex, :delta_crdt, "0.6.5", "c7bb8c2c7e60f59e46557ab4e0224f67ba22f04c02826e273738f3dcc4767adc", [:mix], [{:merkle_map, "~> 0.2.0", [hex: :merkle_map, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c6ae23a525d30f96494186dd11bf19ed9ae21d9fe2c1f1b217d492a7cc7294ae"}, + "ecto": {:hex, :ecto, "3.12.2", "bae2094f038e9664ce5f089e5f3b6132a535d8b018bd280a485c2f33df5c0ce1", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "492e67c70f3a71c6afe80d946d3ced52ecc57c53c9829791bfff1830ff5a1f0c"}, + "ecto_sql": {:hex, :ecto_sql, "3.12.0", "73cea17edfa54bde76ee8561b30d29ea08f630959685006d9c6e7d1e59113b7d", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dc9e4d206f274f3947e96142a8fdc5f69a2a6a9abb4649ef5c882323b6d512f0"}, "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, - "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, - "ecto_sql": {:hex, :ecto_sql, "3.11.3", "4eb7348ff8101fbc4e6bbc5a4404a24fecbe73a3372d16569526b0cf34ebc195", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e5f36e3d736b99c7fee3e631333b8394ade4bafe9d96d35669fca2d81c2be928"}, - "ecto_sqlite3": {:hex, :ecto_sqlite3, "0.13.0", "0c3dc8ff24f378ef108619fd5c18bbbea43cb86dc8733c1c596bd7e0a5bb9e28", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.11", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:exqlite, "~> 0.9", [hex: :exqlite, repo: "hexpm", optional: false]}], "hexpm", "8ab7d8bf6663b811b80c9fa8730780f7077106c40a3fdbae384fe8f82315b257"}, "ed25519": {:hex, :ed25519, "1.4.1", "479fb83c3e31987c9cad780e6aeb8f2015fb5a482618cdf2a825c9aff809afc4", [:mix], [], "hexpm", "0dacb84f3faa3d8148e81019ca35f9d8dcee13232c32c9db5c2fb8ff48c80ec7"}, "elixir_make": {:hex, :elixir_make, "0.7.8", "505026f266552ee5aabca0b9f9c229cbb496c689537c9f922f3eb5431157efc7", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "7a71945b913d37ea89b06966e1342c85cfe549b15e6d6d081e8081c493062c07"}, "equivalex": {:hex, :equivalex, "1.0.3", "170d9a82ae066e0020dfe1cf7811381669565922eb3359f6c91d7e9a1124ff74", [:mix], [], "hexpm", "46fa311adb855117d36e461b9c0ad2598f72110ad17ad73d7533c78020e045fc"}, "ex_doc": {:hex, :ex_doc, "0.35.1", "de804c590d3df2d9d5b8aec77d758b00c814b356119b3d4455e4b8a8687aecaf", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2121c6402c8d44b05622677b761371a759143b958c6c19f6558ff64d0aed40df"}, - "exqlite": {:hex, :exqlite, "0.19.0", "0f3ee29e35bed38552dd0ed59600aa81c78f867f5b5ff0e17d330148e0465483", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "55a8fbb0443f03d4a256e3458bd1203eff5037a6624b76460eaaa9080f462b06"}, "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"}, "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, "flame": {:hex, :flame, "0.5.2", "d46c4daa19b8921b71e0e57dc69edc01ce1311b1976c160192b05d4253b336e8", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, ">= 0.0.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "82560ebef6ab3c277875493d0c93494740c930db0b1a3ff1a570eee9206cc6c0"}, @@ -60,7 +58,7 @@ "merkle_map": {:hex, :merkle_map, "0.2.1", "01a88c87a6b9fb594c67c17ebaf047ee55ffa34e74297aa583ed87148006c4c8", [:mix], [], "hexpm", "fed4d143a5c8166eee4fa2b49564f3c4eace9cb252f0a82c1613bba905b2d04d"}, "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, "mint": {:hex, :mint, "1.6.2", "af6d97a4051eee4f05b5500671d47c3a67dac7386045d87a904126fd4bbcea2e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5ee441dffc1892f1ae59127f74afe8fd82fda6587794278d924e4d90ea3d63f9"}, - "mint_web_socket": {:hex, :mint_web_socket, "1.0.3", "aab42fff792a74649916236d0b01f560a0b3f03ca5dea693c230d1c44736b50e", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "ca3810ca44cc8532e3dce499cc17f958596695d226bb578b2fbb88c09b5954b0"}, + "mint_web_socket": {:hex, :mint_web_socket, "1.0.4", "0b539116dbb3d3f861cdf5e15e269a933cb501c113a14db7001a3157d96ffafd", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "027d4c5529c45a4ba0ce27a01c0f35f284a5468519c045ca15f43decb360a991"}, "mnesiac": {:hex, :mnesiac, "0.3.14", "5ea3f1f3e615073629d0822bcf2297be73149beee2d1f7e482c1943894f59b53", [:mix], [{:libcluster, "~> 3.3", [hex: :libcluster, repo: "hexpm", optional: true]}], "hexpm", "e51b38bf983b9320aba56d5dce79dbf50cbff07f7495e70b89eb45461b8d32fa"}, "myxql": {:hex, :myxql, "0.7.1", "7c7b75aa82227cd2bc9b7fbd4de774fb19a1cdb309c219f411f82ca8860f8e01", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:geo, "~> 3.4", [hex: :geo, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a491cdff53353a09b5850ac2d472816ebe19f76c30b0d36a43317a67c9004936"}, "nebulex": {:hex, :nebulex, "2.6.4", "4b00706e0e676474783d988962abf74614480e13c0a32645acb89bb32b660e09", [:mix], [{:decorator, "~> 1.4", [hex: :decorator, repo: "hexpm", optional: true]}, {:shards, "~> 1.1", [hex: :shards, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "25bdabf3fb86035c8151bba60bda20f80f96ae0261db7bd4090878ff63b03581"}, @@ -77,33 +75,30 @@ "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, "phoenix_pubsub_nats": {:hex, :phoenix_pubsub_nats, "0.2.2", "aedfbda3552299a399cc5d1486f05c313f9eb81e0364e9916e6b3b9ffb40ff41", [:mix], [{:gnat, "~> 1.6", [hex: :gnat, repo: "hexpm", optional: false]}, {:jason, "~> 1.3", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}], "hexpm", "63d117a379f5cc6ba3f9b61a322f821365d3a9b197e43243e0e3b7e47b462a7d"}, - "plug": {:hex, :plug, "1.16.0", "1d07d50cb9bb05097fdf187b31cf087c7297aafc3fed8299aac79c128a707e47", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cbf53aa1f5c4d758a7559c0bd6d59e286c2be0c6a1fac8cc3eee2f638243b93e"}, + "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, "pluggable": {:hex, :pluggable, "1.1.0", "7eba3bc70c0caf4d9056c63c882df8862f7534f0145da7ab3a47ca73e4adb1e4", [:mix], [], "hexpm", "d12eb00ea47b21e92cd2700d6fbe3737f04b64e71b63aad1c0accde87c751637"}, "poly1305": {:hex, :poly1305, "1.0.4", "7cdc8961a0a6e00a764835918cdb8ade868044026df8ef5d718708ea6cc06611", [:mix], [{:chacha20, "~> 1.0", [hex: :chacha20, repo: "hexpm", optional: false]}, {:equivalex, "~> 1.0", [hex: :equivalex, repo: "hexpm", optional: false]}], "hexpm", "e14e684661a5195e149b3139db4a1693579d4659d65bba115a307529c47dbc3b"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, - "postgrex": {:hex, :postgrex, "0.17.4", "5777781f80f53b7c431a001c8dad83ee167bcebcf3a793e3906efff680ab62b3", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "6458f7d5b70652bc81c3ea759f91736c16a31be000f306d3c64bcdfe9a18b3cc"}, + "postgrex": {:hex, :postgrex, "0.19.1", "73b498508b69aded53907fe48a1fee811be34cc720e69ef4ccd568c8715495ea", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "8bac7885a18f381e091ec6caf41bda7bb8c77912bb0e9285212829afe5d8a8f8"}, "protobuf": {:hex, :protobuf, "0.13.0", "7a9d9aeb039f68a81717eb2efd6928fdf44f03d2c0dfdcedc7b560f5f5aae93d", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "21092a223e3c6c144c1a291ab082a7ead32821ba77073b72c68515aa51fef570"}, "protobuf_generate": {:hex, :protobuf_generate, "0.1.3", "57841bc60e2135e190748119d83f78669ee7820c0ad6555ada3cd3cd7df93143", [:mix], [{:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "dae4139b00ba77a279251a0ceb5593b1bae745e333b4ce1ab7e81e8e4906016b"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, - "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"}, "retry": {:hex, :retry, "0.18.0", "dc58ebe22c95aa00bc2459f9e0c5400e6005541cf8539925af0aa027dc860543", [:mix], [], "hexpm", "9483959cc7bf69c9e576d9dfb2b678b71c045d3e6f39ab7c9aa1489df4492d73"}, "salsa20": {:hex, :salsa20, "1.0.4", "404cbea1fa8e68a41bcc834c0a2571ac175580fec01cc38cc70c0fb9ffc87e9b", [:mix], [], "hexpm", "745ddcd8cfa563ddb0fd61e7ce48d5146279a2cf7834e1da8441b369fdc58ac6"}, "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.1", "b8ca910c11429748d3c2d86f0e095abc6d0c49779c7fc5ac5db195e121c46a91", [:mix], [{:ecto, ">= 3.3.0 and < 3.12.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "f5c2f7db1fdcdfe9583ba378689c6ac20fd2af2c476378a017c03c950ac82c3e"}, "shards": {:hex, :shards, "1.1.1", "8b42323457d185b26b15d05187784ce6c5d1e181b35c46fca36c45f661defe02", [:make, :rebar3], [], "hexpm", "169a045dae6668cda15fbf86d31bf433d0dbbaec42c8c23ca4f8f2d405ea8eda"}, - "sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, - "tds": {:hex, :tds, "2.3.5", "fedfb96d53206f01eac62ead859e47e1541a62e1553e9eb7a8801c7dca59eae8", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "52e350f5dd5584bbcff9859e331be144d290b41bd4c749b936014a17660662f2"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"}, "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"}, "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, + "tls_certificate_check": {:hex, :tls_certificate_check, "1.23.0", "bb7869c629de4ec72d4652520c1ad2255bb5712ad09a6568c41b0294b3cec78f", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "79d0c84effc7c81ac1e85fa38b1c33572fe2976fb8fafdfb2f0140de0442d494"}, + "x509": {:hex, :x509, "0.8.9", "03c47e507171507d3d3028d802f48dd575206af2ef00f764a900789dfbe17476", [:mix], [], "hexpm", "ea3fb16a870a199cb2c45908a2c3e89cc934f0434173dc0c828136f878f11661"}, "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"}, - "tls_certificate_check": {:hex, :tls_certificate_check, "1.22.1", "0f450cc1568a67a65ce5e15df53c53f9a098c3da081c5f126199a72505858dc1", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3092be0babdc0e14c2e900542351e066c0fa5a9cf4b3597559ad1e67f07938c0"}, "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm", "c790593b4c3b601f5dc2378baae7efaf5b3d73c4c6456ba85759905be792f2ac"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, - "x509": {:hex, :x509, "0.8.8", "aaf5e58b19a36a8e2c5c5cff0ad30f64eef5d9225f0fd98fb07912ee23f7aba3", [:mix], [], "hexpm", "ccc3bff61406e5bb6a63f06d549f3dba3a1bbb456d84517efaaa210d8a33750f"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, - "yaml_elixir": {:hex, :yaml_elixir, "2.9.0", "9a256da867b37b8d2c1ffd5d9de373a4fda77a32a45b452f1708508ba7bbcb53", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0cb0e7d4c56f5e99a6253ed1a670ed0e39c13fc45a6da054033928607ac08dfc"}, + "yaml_elixir": {:hex, :yaml_elixir, "2.11.0", "9e9ccd134e861c66b84825a3542a1c22ba33f338d82c07282f4f1f52d847bd50", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "53cc28357ee7eb952344995787f4bb8cc3cecbf189652236e9b163e8ce1bc242"}, } diff --git a/spawn_statestores/statestore_controller/.formatter.exs b/spawn_statestores/statestore_controller/.formatter.exs new file mode 100644 index 00000000..1f7490ba --- /dev/null +++ b/spawn_statestores/statestore_controller/.formatter.exs @@ -0,0 +1,6 @@ +# Used by "mix format" +[ + import_deps: [:ecto], + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"], + subdirectories: ["priv/*/migrations"] +] diff --git a/spawn_statestores/statestore_controller/.gitignore b/spawn_statestores/statestore_controller/.gitignore new file mode 100644 index 00000000..fe54b263 --- /dev/null +++ b/spawn_statestores/statestore_controller/.gitignore @@ -0,0 +1,26 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +statestores-*.tar + +# Temporary files, for example, from tests. +/tmp/ diff --git a/spawn_statestores/statestore_controller/LICENSE b/spawn_statestores/statestore_controller/LICENSE new file mode 100644 index 00000000..725bd897 --- /dev/null +++ b/spawn_statestores/statestore_controller/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Eigr + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/spawn_statestores/statestore_controller/README.md b/spawn_statestores/statestore_controller/README.md new file mode 100644 index 00000000..be297d62 --- /dev/null +++ b/spawn_statestores/statestore_controller/README.md @@ -0,0 +1,23 @@ +# Statestores + + + +Spawn Statestores MariaDB is a storage lib for the Spawn Actors System + +## Installation + +[Available in Hex](https://hex.pm/packages/spawn_statestores), the package can be installed +by adding `statestores` to your list of dependencies in `mix.exs`: + +```elixir +def deps do + [ + {:spawn_statestores_mariadb, "~> 0.1.0"} + ] +end +``` + +Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) +and published on [HexDocs](https://hexdocs.pm). Once published, the docs can +be found at . + diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller.ex b/spawn_statestores/statestore_controller/lib/statestore_controller.ex new file mode 100644 index 00000000..e69de29b diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/application.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/application.ex new file mode 100644 index 00000000..f1dee065 --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/application.ex @@ -0,0 +1,14 @@ +defmodule StatestoreController.Application do + @moduledoc false + use Application + + @impl true + def start(_type, _args) do + children = [ + {StatestoreController.Supervisor, []} + ] + + opts = [strategy: :one_for_one, name: StatestoreController.Supervisor] + Supervisor.start_link(children, opts) + end +end diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/cdc_supervisor.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/cdc_supervisor.ex new file mode 100644 index 00000000..fe903f15 --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/cdc_supervisor.ex @@ -0,0 +1,36 @@ +defmodule StatestoreController.CDC.CdcSupervisor do + @moduledoc false + use Supervisor + require Logger + + alias StatestoreController.CDC.Postgres.Replication + alias StatestoreController.CDC.Postgres.MessageHandler + + def child_spec(opts) do + %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [opts]}, + shutdown: 120_000 + } + end + + @spec start_link(any) :: :ignore | {:error, any} | {:ok, pid} + def start_link(opts) do + Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + @impl true + def init(opts) do + with {:ok, _pg} <- Postgrex.start_link(opts), + {:ok, _cdc_pid} <- Replication.start_link(opts) do + children = [ + {MessageHandler, opts} + ] + + Supervisor.init(children, strategy: :rest_for_one) + else + _ -> + Supervisor.init([], strategy: :rest_for_one) + end + end +end diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/messaging/kafka_producer.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/messaging/kafka_producer.ex new file mode 100644 index 00000000..d89a4a03 --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/messaging/kafka_producer.ex @@ -0,0 +1,32 @@ +defmodule StatestoreController.CDC.Messaging.KafkaProducer do + @moduledoc false + require Logger + + @behaviour StatestoreController.CDC.Producer + + @impl true + @spec is_alive?() :: boolean() + def is_alive?() do + KafkaEx.metadata() + true + catch + :exit, details -> + Logger.error("Kafka broker is down or we can't connect to it!. #{inspect(details)}") + + false + end + + @impl true + @spec produce(String.t(), any(), Keyword.t()) :: + nil + | :ok + | {:ok, integer} + | {:error, :closed} + | {:error, :inet.posix()} + | {:error, any} + | :leader_not_available + | {:error, :cannot_produce, any()} + def produce(topic, data, opts \\ []) do + nil + end +end diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/message_handler.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/message_handler.ex new file mode 100644 index 00000000..4c57d0cd --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/message_handler.ex @@ -0,0 +1,108 @@ +defmodule StatestoreController.CDC.Postgres.MessageHandler do + @moduledoc """ + + """ + use GenServer + require Logger + + alias StatestoreController.CDC.Postgres.Tx + alias StatestoreController.CDC.Messaging.Producer + + @impl true + def init(opts) do + topic = Keyword.fetch!(opts, :sink_topic) + tables = Keyword.fetch!(opts, :source_tables) + {:ok, %{tables: tables, topic: topic}} + end + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @impl true + def handle_cast( + {:handle, %Tx{timestamp: timestamp, operations: operations} = transaction} = _msg, + %{tables: tables, topic: topic} = state + ) do + Logger.debug("#{inspect(__MODULE__)} Received message: #{inspect(transaction)}") + + case Producer.alive?() do + true -> + Enum.filter(operations, fn op -> + cond do + is_nil(tables) -> + true + + length(tables) > 0 -> + Enum.member?(tables, op.table) + + true -> + true + end + end) + |> Enum.map(fn op -> + to_data(timestamp, op) + |> produces(topic, []) + end) + + _ -> + {:error, :unavailable, %{status: "Error Kafka is not available"}} + end + + {:noreply, state} + end + + def handle_cast(msg, state) do + Logger.warn("Received an unexpected message: #{inspect(msg)}") + {:noreply, state} + end + + def publish(tx) do + GenServer.cast(__MODULE__, {:handle, tx}) + end + + defp to_data( + timestamp, + %StatestoreController.CDC.Postgres.Tx.Operation{ + type: op, + namespace: namespace, + table: table, + record: record, + old_record: old + } = _operation + ) do + %{ + "timestamp" => DateTime.to_string(timestamp), + "operation" => Atom.to_string(op), + "schema" => namespace, + "table" => table, + "data_before" => old, + "data_after" => record + } + end + + defp produces(payload, topic, opts) do + case Producer.produce(topic, payload, opts) do + :ok -> + {:ok, %{status: "Ok"}} + + {:ok, partition} -> + {:ok, %{status: "Ok", partition: partition}} + + {:error, :closed} -> + {:error, :closed, %{status: "Error. Connection closed"}} + + :leader_not_available -> + {:error, :leader_not_available, %{status: "Error. Kafka Node Leader is not available"}} + + {:error, :cannot_produce, error} -> + {:error, :cannot_produce, %{status: "Error. #{inspect(error)}"}} + + {:error, error} -> + {:error, %{status: "Error. #{inspect(error)}"}} + + error -> + {:error, %{status: "Error. #{inspect(error)}"}} + end + end +end diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/protocol.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/protocol.ex new file mode 100644 index 00000000..04485b2e --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/protocol.ex @@ -0,0 +1,73 @@ +defmodule StatestoreController.CDC.Postgres.Protocol do + @moduledoc """ + + """ + require Logger + import Postgrex.PgOutput.Messages + + alias StatestoreController.CDC.Postgres.Protocol.Tx + alias Postgrex.PgOutput.Lsn + + @type t :: %__MODULE__{ + tx: Tx.t(), + relations: map() + } + + defstruct [ + :tx, + relations: %{} + ] + + @spec new() :: t() + def new do + %__MODULE__{} + end + + def handle_message(msg, state) when is_binary(msg) do + msg + |> decode() + |> handle_message(state) + end + + def handle_message(msg_primary_keep_alive(reply: 0), state), do: {[], nil, state} + + def handle_message(msg_primary_keep_alive(server_wal: lsn, reply: 1), state) do + Logger.debug("msg_primary_keep_alive message reply=true") + <> = Lsn.encode(lsn) + + {[standby_status_update(lsn)], nil, state} + end + + def handle_message(msg, %__MODULE__{tx: nil, relations: relations} = state) do + tx = + [relations: relations, decode: true] + |> Tx.new() + |> Tx.build(msg) + + {[], nil, %{state | tx: tx}} + end + + def handle_message(msg, %__MODULE__{tx: tx} = state) do + case Tx.build(tx, msg) do + %Tx{state: :commit, relations: relations} -> + tx = Tx.finalize(tx) + relations = Map.merge(state.relations, relations) + {[], tx, %{state | tx: nil, relations: relations}} + + tx -> + {[], nil, %{state | tx: tx}} + end + end + + defp standby_status_update(lsn) do + [ + wal_recv: lsn + 1, + wal_flush: lsn + 1, + wal_apply: lsn + 1, + system_clock: now(), + reply: 0 + ] + |> msg_standby_status_update() + |> encode() + end +end diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/replication/replication.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/replication/replication.ex new file mode 100644 index 00000000..011cde12 --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/replication/replication.ex @@ -0,0 +1,97 @@ +defmodule StatestoreController.CDC.Postgres.Replication do + @moduledoc """ + `Replication` is the process responsible for replicating database information. + """ + use Postgrex.ReplicationConnection + alias StatestoreController.CDC.Postgres.Protocol + alias StatestoreController.CDC.Postgres.MessageHandler + + require Logger + + defstruct [ + :publications, + :protocol, + :slot, + :state, + subscribers: %{} + ] + + @impl true + def init({slot, pubs}) do + {:ok, + %__MODULE__{ + slot: slot, + publications: pubs, + protocol: Protocol.new() + }} + end + + def start_link(opts) do + conn_opts = [auto_reconnect: true] + publications = opts[:publications] || raise ArgumentError, message: "`:publications` missing" + slot = opts[:slot] || raise ArgumentError, message: "`:slot` missing" + + Postgrex.ReplicationConnection.start_link( + __MODULE__, + {slot, publications}, + conn_opts ++ opts + ) + end + + @impl true + def handle_connect(%__MODULE__{slot: slot} = state) do + query = "CREATE_REPLICATION_SLOT #{slot} TEMPORARY LOGICAL pgoutput NOEXPORT_SNAPSHOT" + + Logger.debug("[create slot] query=#{query}") + + {:query, query, %{state | state: :create_slot}} + end + + @impl true + def handle_result( + [%Postgrex.Result{} | _], + %__MODULE__{state: :create_slot, publications: pubs, slot: slot} = state + ) do + opts = [proto_version: 1, publication_names: pubs] + + query = "START_REPLICATION SLOT #{slot} LOGICAL 0/0 #{escape_options(opts)}" + + Logger.debug("[Start streaming] query=#{query}") + + {:stream, query, [], %{state | state: :streaming}} + end + + @impl true + def handle_data(msg, state) do + {return_msgs, tx, protocol} = Protocol.handle_message(msg, state.protocol) + + if not is_nil(tx) do + Logger.info( + "Publish transaction evento to MessageHandler for processing. Tx: #{inspect(tx)}" + ) + + MessageHandler.publish(tx) + end + + {:noreply, return_msgs, %{state | protocol: protocol}} + end + + @impl true + def handle_info({:DOWN, _ref, :process, _, _}, state) do + {:noreply, state} + end + + defp escape_options([]), + do: "" + + defp escape_options(opts) do + parts = + Enum.map_intersperse(opts, ", ", fn {k, v} -> [Atom.to_string(k), ?\s, escape_string(v)] end) + + [?\s, ?(, parts, ?)] + end + + defp escape_string(value) do + [?', :binary.replace(to_string(value), "'", "''", [:global]), ?'] + end +end diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/tx.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/tx.ex new file mode 100644 index 00000000..cbd24f7a --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/tx.ex @@ -0,0 +1,78 @@ +defmodule StatestoreController.CDC.Postgres.Tx do + @moduledoc """ + + """ + alias Postgrex.PgOutput.Lsn + + import Postgrex.PgOutput.Messages + + alias StatestoreController.CDC.Postgres.Tx.Operation + + @type t :: %__MODULE__{ + operations: [Operation.t()], + relations: map(), + timestamp: term(), + xid: pos_integer(), + state: :begin | :commit, + lsn: Lsn.t(), + end_lsn: Lsn.t() + } + + defstruct [ + :timestamp, + :xid, + :lsn, + :end_lsn, + relations: %{}, + operations: [], + state: :begin, + decode: true + ] + + def new(opts \\ []) do + struct(__MODULE__, opts) + end + + def finalize(%__MODULE__{state: :commit, operations: ops} = tx) do + %{tx | operations: Enum.reverse(ops)} + end + + def finalize(%__MODULE__{} = tx), do: tx + + @spec build(t(), tuple()) :: t() + def build(tx, msg_xlog_data(data: data)) do + build(tx, data) + end + + def build(tx, msg_begin(lsn: lsn, timestamp: ts, xid: xid)) do + %{tx | lsn: lsn, timestamp: ts, xid: xid, state: :begin} + end + + def build(%__MODULE__{state: :begin, relations: relations} = tx, msg_relation(id: id) = rel) do + %{tx | relations: Map.put(relations, id, rel)} + end + + def build(%__MODULE__{state: :begin, lsn: tx_lsn} = tx, msg_commit(lsn: lsn, end_lsn: end_lsn)) + when tx_lsn == lsn do + %{tx | state: :commit, end_lsn: end_lsn} + end + + def build(%__MODULE__{state: :begin} = builder, msg_insert(relation_id: id) = msg), + do: build_op(builder, id, msg) + + def build(%__MODULE__{state: :begin} = builder, msg_update(relation_id: id) = msg), + do: build_op(builder, id, msg) + + def build(%__MODULE__{state: :begin} = builder, msg_delete(relation_id: id) = msg), + do: build_op(builder, id, msg) + + # skip unknown messages + def build(%__MODULE__{} = tx, _msg), do: tx + + defp build_op(%__MODULE__{state: :begin, relations: rels, decode: decode} = tx, id, msg) do + rel = Map.fetch!(rels, id) + op = Operation.from_msg(msg, rel, decode) + + %{tx | operations: [op | tx.operations]} + end +end diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/tx/operation.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/tx/operation.ex new file mode 100644 index 00000000..8cacb349 --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/postgres/tx/operation.ex @@ -0,0 +1,93 @@ +defmodule StatestoreController.CDC.Postgres.Tx.Operation do + @moduledoc """ + `Describes` a change (INSERT, UPDATE, DELETE) within a transaction. + """ + + import Postgrex.PgOutput.Messages + alias Postgrex.PgOutput.Type, as: PgType + + @type t :: %__MODULE__{} + defstruct [ + :type, + :schema, + :namespace, + :table, + :record, + :old_record, + :timestamp + ] + + @spec from_msg(tuple(), tuple(), decode :: boolean()) :: t() + def from_msg( + msg_insert(data: data), + msg_relation(columns: columns, namespace: ns, name: name), + decode? + ) do + %__MODULE__{ + type: :insert, + namespace: ns, + schema: into_schema(columns), + table: name, + record: cast(data, columns, decode?), + old_record: %{} + } + end + + def from_msg( + msg_update(change_data: data, old_data: old_data), + msg_relation(columns: columns, namespace: ns, name: name), + decode? + ) do + %__MODULE__{ + type: :update, + namespace: ns, + table: name, + schema: into_schema(columns), + record: cast(data, columns, decode?), + old_record: cast(columns, old_data, decode?) + } + end + + def from_msg( + msg_delete(old_data: data), + msg_relation(columns: columns, namespace: ns, name: name), + decode? + ) do + %__MODULE__{ + type: :delete, + namespace: ns, + schema: into_schema(columns), + table: name, + record: %{}, + old_record: cast(data, columns, decode?) + } + end + + defp into_schema(columns) do + for c <- columns do + c + |> column() + |> Enum.into(%{}) + end + end + + defp cast(data, columns, decode?) do + Enum.zip_reduce([data, columns], %{}, fn [text, typeinfo], acc -> + key = column(typeinfo, :name) + + value = + if decode? do + t = + typeinfo + |> column(:type) + |> PgType.type_info() + + PgType.decode(text, t) + else + text + end + + Map.put(acc, key, value) + end) + end +end diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/producer.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/producer.ex new file mode 100644 index 00000000..a66c74bc --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/cdc/producer.ex @@ -0,0 +1,15 @@ +defmodule StatestoreController.CDC.Producer do + @moduledoc """ + + """ + @callback is_alive?() :: boolean() + @callback produce(String.t(), any(), Keyword.t()) :: + nil + | :ok + | {:ok, integer} + | {:error, :closed} + | {:error, :inet.posix()} + | {:error, any} + | :leader_not_available + | {:error, :cannot_produce, any()} +end diff --git a/spawn_statestores/statestore_controller/lib/statestore_controller/supervisor.ex b/spawn_statestores/statestore_controller/lib/statestore_controller/supervisor.ex new file mode 100644 index 00000000..cbc6d615 --- /dev/null +++ b/spawn_statestores/statestore_controller/lib/statestore_controller/supervisor.ex @@ -0,0 +1,43 @@ +defmodule StatestoreController.Supervisor do + @moduledoc false + + import Statestores.Util, + only: [load_lookup_adapter: 0, load_snapshot_adapter: 0] + + def start_link(args) do + Supervisor.start_link(__MODULE__, args, name: __MODULE__) + end + + def child_spec(args) do + %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [args]} + } + end + + @impl true + def init(args) do + lookup_adapter = load_lookup_adapter() + snapshot_adapter = load_snapshot_adapter() + Statestores.Migrator.migrate(snapshot_adapter) + Statestores.Migrator.migrate(lookup_adapter) + + children = + [ + Statestores.Vault, + snapshot_adapter, + lookup_adapter + ] + |> maybe_add_cdc(snapshot_adapter, args) + + Supervisor.init(children, strategy: :one_for_one) + end + + defp maybe_add_cdc(children, snapshot_adapter, args) + when is_atom(snapshot_adapter) and + snapshot_adapter in [Statestores.Adapters.PostgresSnapshotAdapter] do + children ++ [{StatestoreController.CDC.CdcSupervisor, [args]}] + end + + defp maybe_add_cdc(_children, _snapshot_adapter, _args), do: nil +end diff --git a/spawn_statestores/statestore_controller/mix.exs b/spawn_statestores/statestore_controller/mix.exs new file mode 100644 index 00000000..605b1061 --- /dev/null +++ b/spawn_statestores/statestore_controller/mix.exs @@ -0,0 +1,69 @@ +defmodule StatestoresMysql.MixProject do + use Mix.Project + + @app :spawn_statestore_controller + @version "0.0.0-local.dev" + @source_url "https://github.com/eigr/spawn/blob/main/spawn_statestores/statestore_controller" + + def project do + [ + app: @app, + version: @version, + description: "Spawn Statestores Controller", + source_url: @source_url, + homepage_url: "https://eigr.io/", + build_path: "../../_build", + config_path: "../../config/config.exs", + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.15", + start_permanent: Mix.env() == :prod, + deps: deps(), + elixirc_paths: elixirc_paths(Mix.env()), + package: package(), + docs: docs() + ] + end + + # Run "mix help compile.app" to learn about applications. + def application do + [ + extra_applications: [:logger] + ] + end + + defp package do + [ + files: ["lib", "mix.exs", "README.md", "LICENSE"], + licenses: ["Apache-2.0"], + links: %{GitHub: @source_url} + ] + end + + defp docs do + [ + main: "readme", + source_url: @source_url, + source_ref: "v#{@version}", + formatter_opts: [gfm: true], + extras: [ + "README.md" + ] + ] + end + + # Run "mix help deps" to learn about dependencies. + defp deps do + [ + {:cloak_ecto, "~> 1.2"}, + {:ecto_sql, "~> 3.10"}, + {:postgrex, "~> 0.19"}, + {:postgrex_pgoutput, "~> 0.1"}, + {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, + {:spawn_statestores, path: "../statestores"} + ] + end + + defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(_), do: ["lib"] +end diff --git a/spawn_statestores/statestore_controller/test/support/data_case.ex b/spawn_statestores/statestore_controller/test/support/data_case.ex new file mode 100644 index 00000000..98e14ef3 --- /dev/null +++ b/spawn_statestores/statestore_controller/test/support/data_case.ex @@ -0,0 +1,13 @@ +defmodule Statestores.DataCase do + @moduledoc false + + use ExUnit.CaseTemplate + + using do + quote do + use Statestores.SandboxHelper, repos: [Statestores.Util.load_snapshot_adapter()] + + import Statestores.DataCase + end + end +end diff --git a/spawn_statestores/statestore_controller/test/support/sandbox_helper.ex b/spawn_statestores/statestore_controller/test/support/sandbox_helper.ex new file mode 100644 index 00000000..f204e650 --- /dev/null +++ b/spawn_statestores/statestore_controller/test/support/sandbox_helper.ex @@ -0,0 +1,23 @@ +defmodule Statestores.SandboxHelper do + @moduledoc false + + defmacro __using__(args) do + quote do + setup _tags do + repos = unquote(args[:repos]) + + Enum.each(repos, fn repo -> + :ok = Ecto.Adapters.SQL.Sandbox.checkout(repo) + Ecto.Adapters.SQL.Sandbox.mode(repo, :auto) + end) + + on_exit(fn -> + Enum.each(repos, fn repo -> + :ok = Ecto.Adapters.SQL.Sandbox.checkout(repo) + Ecto.Adapters.SQL.Sandbox.mode(repo, :auto) + end) + end) + end + end + end +end diff --git a/spawn_statestores/statestore_controller/test/test_helper.exs b/spawn_statestores/statestore_controller/test/test_helper.exs new file mode 100644 index 00000000..16f8bc05 --- /dev/null +++ b/spawn_statestores/statestore_controller/test/test_helper.exs @@ -0,0 +1,15 @@ +Application.put_env( + :spawn_statestores, + :database_adapter, + Statestores.Adapters.MariaDBSnapshotAdapter +) + +Application.put_env( + :spawn_statestores, + :database_lookup_adapter, + Statestores.Adapters.MariaDBLookupAdapter +) + +ExUnit.start() + +Statestores.Supervisor.start_link(%{}) diff --git a/spawn_statestores/statestores/lib/statestores/supervisor.ex b/spawn_statestores/statestores/lib/statestores/supervisor.ex index 2a747e22..28732490 100644 --- a/spawn_statestores/statestores/lib/statestores/supervisor.ex +++ b/spawn_statestores/statestores/lib/statestores/supervisor.ex @@ -27,9 +27,15 @@ defmodule Statestores.Supervisor do def init(_args) do lookup_adapter = load_lookup_adapter() snapshot_adapter = load_snapshot_adapter() - projection_adapter = load_projection_adapter() - Statestores.Migrator.migrate(snapshot_adapter) - Statestores.Migrator.migrate(lookup_adapter) + + case System.get_env("MIX_ENV") do + env when env in ["dev", "test"] -> + Statestores.Migrator.migrate(snapshot_adapter) + Statestores.Migrator.migrate(lookup_adapter) + + _ -> + nil + end children = [ diff --git a/spawn_statestores/statestores_postgres/mix.exs b/spawn_statestores/statestores_postgres/mix.exs index e546ac40..4cc646d0 100644 --- a/spawn_statestores/statestores_postgres/mix.exs +++ b/spawn_statestores/statestores_postgres/mix.exs @@ -58,7 +58,7 @@ defmodule StatestoresPostgres.MixProject do {:cloak_ecto, "~> 1.2"}, {:ecto_sql, "~> 3.10"}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, - {:postgrex, "~> 0.16"}, + {:postgrex, "~> 0.17"}, {:spawn_statestores, path: "../statestores"} ] end