diff --git a/Cargo.lock b/Cargo.lock index e8c3910514..aad5950542 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,6 +144,15 @@ version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -186,9 +195,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.87" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d556ec1359574147ec0c4fc5eb525f3f23263a592b1a9c07e0a75b427de55c97" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", @@ -812,6 +821,8 @@ dependencies = [ "tempfile", "tokio", "toml", + "tracing", + "tracing-subscriber", ] [[package]] @@ -1818,11 +1829,14 @@ dependencies = [ "jsonrpc-core", "log", "nix", + "postgresql_embedded", "rand 0.8.5", "reqwest", "serde_json", "tempfile", "toml", + "tracing", + "tracing-subscriber", ] [[package]] @@ -2503,6 +2517,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", +] + [[package]] name = "derive_more" version = "0.99.19" @@ -2761,6 +2786,18 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "findshlibs" version = "0.10.2" @@ -3040,8 +3077,8 @@ dependencies = [ "aho-corasick", "bstr", "log", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", + "regex-automata", + "regex-syntax", ] [[package]] @@ -3702,7 +3739,7 @@ dependencies = [ "globset", "log", "memchr", - "regex-automata 0.4.9", + "regex-automata", "same-file", "walkdir", "winapi-util", @@ -3796,6 +3833,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -3985,12 +4025,43 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "liblzma" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a631d2b24be269775ba8f7789a6afa1ac228346a20c9e87dbbbe4975a79fd764" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efdadf1a99aceff34553de1461674ab6ac7e7f0843ae9875e339f4a14eb43475" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +[[package]] +name = "libredox" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488594b9328dee448adb906d8b126d9b7deb7cf5c22161ee591610bb1be83c0" +dependencies = [ + "bitflags 2.9.0", + "libc", + "redox_syscall 0.5.10", +] + [[package]] name = "libsqlite3-sys" version = "0.30.1" @@ -4017,6 +4088,12 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + [[package]] name = "litemap" version = "0.7.5" @@ -4065,11 +4142,11 @@ checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -4278,6 +4355,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" +[[package]] +name = "nu-ansi-term" +version = "0.50.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -4311,6 +4397,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -4901,6 +4997,67 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +[[package]] +name = "postgresql_archive" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f799d21d3a7225f5efb33449b97046135244b46a3d963faa61a888816df2c85a" +dependencies = [ + "async-trait", + "flate2", + "futures-util", + "hex", + "liblzma", + "num-format", + "regex-lite", + "reqwest", + "reqwest-middleware", + "reqwest-retry", + "reqwest-tracing", + "semver", + "serde", + "serde_json", + "sha2", + "tar", + "target-triple", + "tempfile", + "thiserror 2.0.12", + "tracing", + "url", + "zip", +] + +[[package]] +name = "postgresql_commands" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44262b2d4a8e7d4da9a178ee9526997d8ee96f3acc619fdd986475aad9354a62" +dependencies = [ + "thiserror 2.0.12", + "tokio", + "tracing", +] + +[[package]] +name = "postgresql_embedded" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a49030a517128776430b69490c9c4b9cd8c8a63b6ef1bc923835dea523e025c5" +dependencies = [ + "anyhow", + "postgresql_archive", + "postgresql_commands", + "rand 0.9.0", + "semver", + "sqlx", + "target-triple", + "tempfile", + "thiserror 2.0.12", + "tokio", + "tracing", + "url", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -5010,7 +5167,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.8.5", + "regex-syntax", "rusty-fork", "tempfile", "unarray", @@ -5269,7 +5426,7 @@ checksum = "9efd944f26afa2406cbbabff39fac533c9bc24b13d7f1f12e14ae3e7bdc66cdb" dependencies = [ "cfg-if", "libc", - "rustix", + "rustix 0.38.44", "windows 0.60.0", ] @@ -5281,17 +5438,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -5302,14 +5450,14 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] -name = "regex-syntax" -version = "0.6.29" +name = "regex-lite" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" @@ -5319,9 +5467,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.12" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ "base64 0.22.1", "bytes", @@ -5353,15 +5501,70 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-util", "tower 0.5.2", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "windows-registry", ] +[[package]] +name = "reqwest-middleware" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e" +dependencies = [ + "anyhow", + "async-trait", + "http 1.2.0", + "reqwest", + "serde", + "thiserror 1.0.69", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "getrandom 0.2.15", + "http 1.2.0", + "hyper", + "parking_lot 0.11.2", + "reqwest", + "reqwest-middleware", + "retry-policies", + "thiserror 1.0.69", + "tokio", + "tracing", + "wasm-timer", +] + +[[package]] +name = "reqwest-tracing" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d70ea85f131b2ee9874f0b160ac5976f8af75f3c9badfe0d955880257d10bd83" +dependencies = [ + "anyhow", + "async-trait", + "getrandom 0.2.15", + "http 1.2.0", + "matchit 0.8.4", + "reqwest", + "reqwest-middleware", + "tracing", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -5372,6 +5575,15 @@ dependencies = [ "quick-error", ] +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "rhai" version = "1.21.0" @@ -5475,7 +5687,20 @@ dependencies = [ "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +dependencies = [ + "bitflags 2.9.0", + "errno", + "libc", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] @@ -5769,9 +5994,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] @@ -5789,9 +6014,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", @@ -5919,6 +6144,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "siphasher" version = "0.3.11" @@ -6379,17 +6610,33 @@ dependencies = [ "libc", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "target-triple" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac9aa371f599d22256307c24a9d748c041e548cbf599f35d890f9d365361790" + [[package]] name = "tempfile" -version = "3.17.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e5a0acb1f3f55f65cc4a866c361b2fb2a0ff6366785ae6fbb5f85df07ba230" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ - "cfg-if", "fastrand", "getrandom 0.3.1", "once_cell", - "rustix", + "rustix 1.0.8", "windows-sys 0.59.0", ] @@ -6507,7 +6754,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" dependencies = [ - "rustix", + "rustix 0.38.44", "windows-sys 0.48.0", ] @@ -6693,6 +6940,7 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", "socket2", @@ -6920,19 +7168,33 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", + "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", + "smallvec", "thread_local", "tracing", "tracing-core", + "tracing-log", ] [[package]] @@ -7320,6 +7582,34 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.77" @@ -7358,7 +7648,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix", + "rustix 0.38.44", ] [[package]] @@ -7458,8 +7748,8 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link", - "windows-result 0.3.1", - "windows-strings 0.3.1", + "windows-result", + "windows-strings", ] [[package]] @@ -7496,9 +7786,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.0" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-numerics" @@ -7512,22 +7802,13 @@ dependencies = [ [[package]] name = "windows-registry" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" -dependencies = [ - "windows-result 0.2.0", - "windows-strings 0.1.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-result" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "windows-targets 0.52.6", + "windows-result", + "windows-strings", + "windows-targets 0.53.3", ] [[package]] @@ -7539,16 +7820,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result 0.2.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-strings" version = "0.3.1" @@ -7609,13 +7880,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -7628,6 +7916,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -7640,6 +7934,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -7652,12 +7952,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -7670,6 +7982,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -7682,6 +8000,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -7694,6 +8018,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -7706,6 +8036,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winreg" version = "0.50.0" @@ -7749,6 +8085,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "xattr" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" +dependencies = [ + "libc", + "rustix 1.0.8", +] + [[package]] name = "xml-rs" version = "0.8.25" @@ -7909,3 +8255,32 @@ dependencies = [ "quote", "syn 2.0.99", ] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap 2.7.1", + "memchr", + "thiserror 2.0.12", + "zopfli", +] + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 90a6c6f685..2097f8a0fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -268,6 +268,7 @@ p2p = { version = "0.7.0", package = "tentacle", default-features = false } parking_lot = "0.12" paste = "1.0" path-clean = "0.1.0" +postgresql_embedded = { version = "0.18", default-features = false, features = ["blocking", "theseus"] } pretty_assertions = "1.3.0" proc-macro2 = "1.0" prometheus = "0.13.3" @@ -310,6 +311,8 @@ yansi = "0.5" seq-macro = "0.3" strum = { version = "0.27", features = ["derive"] } phf = { version = "0.12", features = ["macros"] } +tracing = { version = "0.1.41", features = ["log", "log-always"] } +tracing-subscriber = "0.3.20" [profile.release] overflow-checks = true diff --git a/Makefile b/Makefile index d5b185d8ac..ed309d37dc 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ CKB_FEATURES ?= deadlock_detection,with_sentry ALL_FEATURES := deadlock_detection,with_sentry,with_dns_seeding,profiling,march-native CKB_BENCH_FEATURES ?= ci CKB_BUILD_TARGET ?= -INTEGRATION_RUST_LOG := info,ckb_test=debug,ckb_sync=debug,ckb_relay=debug,ckb_network=debug +INTEGRATION_RUST_LOG := info CARGO_TARGET_DIR ?= $(shell pwd)/target BINARY_NAME ?= "ckb" COV_PROFRAW_DIR = ${CARGO_TARGET_DIR}/cov diff --git a/ckb-bin/Cargo.toml b/ckb-bin/Cargo.toml index f376d7b9cc..d0c0f082f1 100644 --- a/ckb-bin/Cargo.toml +++ b/ckb-bin/Cargo.toml @@ -47,6 +47,8 @@ is-terminal.workspace = true fdlimit.workspace = true ckb-stop-handler.workspace = true tokio = { workspace = true, features = ["sync"] } +tracing.workspace = true +tracing-subscriber.workspace = true [target.'cfg(not(target_os="windows"))'.dependencies] daemonize = { version = "0.5.0" } diff --git a/ckb-bin/src/lib.rs b/ckb-bin/src/lib.rs index 5f8f9972c9..6ae25e724f 100644 --- a/ckb-bin/src/lib.rs +++ b/ckb-bin/src/lib.rs @@ -42,6 +42,7 @@ pub fn run_app(version: Version) -> Result<(), ExitCode> { unsafe { ::std::env::set_var("RUST_BACKTRACE", "full"); } + // tracing_subscriber::fmt::init(); let (bin_name, app_matches) = cli::get_bin_name_and_matches(&version); if let Some((cli, matches)) = app_matches.subcommand() { @@ -133,6 +134,7 @@ fn run_app_inner( let setup = Setup::from_matches(bin_name, cmd, matches)?; let _guard = SetupGuard::from_setup(&setup, &version, handle.clone(), is_silent_logging)?; + info!("run app inner............................."); raise_fd_limit(); let ret = match cmd { diff --git a/rpc/src/service_builder.rs b/rpc/src/service_builder.rs index 0435806301..cfed541cda 100644 --- a/rpc/src/service_builder.rs +++ b/rpc/src/service_builder.rs @@ -13,6 +13,7 @@ use ckb_app_config::{DBConfig, IndexerConfig, RpcConfig}; use ckb_chain::ChainController; use ckb_indexer::IndexerService; use ckb_indexer_sync::{PoolService, new_secondary_db}; +use ckb_logger::{debug, info}; use ckb_network::NetworkController; use ckb_network_alert::{notifier::Notifier as AlertNotifier, verifier::Verifier as AlertVerifier}; use ckb_pow::Pow; @@ -227,6 +228,8 @@ impl<'a> ServiceBuilder<'a> { if self.config.rich_indexer_enable() { // Init rich-indexer service + info!("Rich Indexer enabled"); + debug!("Rich Indexer enabled"); let mut rich_indexer = RichIndexerService::new( ckb_secondary_db, pool_service, diff --git a/test/Cargo.toml b/test/Cargo.toml index a8e1c76964..1910fcb897 100644 --- a/test/Cargo.toml +++ b/test/Cargo.toml @@ -4,7 +4,7 @@ version.workspace = true license = "MIT" authors = ["Nervos Core Dev "] edition = "2024" -description = "CKB integration tests." +description = "ckb integration tests." homepage = "https://github.com/nervosnetwork/ckb" repository = "https://github.com/nervosnetwork/ckb" @@ -40,6 +40,9 @@ byteorder.workspace = true jsonrpc-core.workspace = true ctrlc.workspace = true log.workspace = true +postgresql_embedded.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true [target.'cfg(not(target_os="windows"))'.dependencies] nix = { version = "0.29.0", default-features = false, features = ["signal"] } diff --git a/test/src/lib.rs b/test/src/lib.rs index 924ab5b8e0..bae0aea439 100644 --- a/test/src/lib.rs +++ b/test/src/lib.rs @@ -68,6 +68,8 @@ pub fn main_test() { env::set_var("RUST_BACKTRACE", "full"); } + // tracing_subscriber::fmt::init(); + let clap_app = clap_app(); let matches = clap_app.get_matches(); @@ -636,6 +638,8 @@ fn all_specs() -> Vec> { Box::new(CheckVmBExtension), Box::new(RandomlyKill), Box::new(SyncChurn), + // Rich-indexer chain reorganization bug reproduction test + Box::new(RichIndexerUncleBlockBug::default()), ]; specs.shuffle(&mut thread_rng()); specs diff --git a/test/src/node.rs b/test/src/node.rs index 8a0684e6ac..f508c1c50d 100644 --- a/test/src/node.rs +++ b/test/src/node.rs @@ -667,7 +667,7 @@ impl Node { "--ba-advanced", ]) .stdin(Stdio::null()) - .stdout(Stdio::null()) + .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) .spawn() .expect("failed to run binary"); @@ -793,18 +793,19 @@ impl Node { pub fn wait_find_unverified_blocks_finished(&self) { // wait for node[0] to find unverified blocks finished - - let now = std::time::Instant::now(); - while !self - .access_log(|line: &str| line.contains("find unverified blocks finished")) - .expect("node[0] must have log") - { - std::thread::sleep(std::time::Duration::from_secs(1)); - if now.elapsed() > Duration::from_secs(60) { - panic!("node[0] should find unverified blocks finished in 60s"); - } - info!("waiting for node[0] to find unverified blocks finished"); - } + std::thread::sleep(Duration::from_secs(5)); + + // let now = std::time::Instant::now(); + // while !self + // .access_log(|line: &str| line.contains("find unverified blocks finished")) + // .expect("node[0] must have log") + // { + // std::thread::sleep(std::time::Duration::from_secs(1)); + // if now.elapsed() > Duration::from_secs(60) { + // panic!("node[0] should find unverified blocks finished in 60s"); + // } + // info!("waiting for node[0] to find unverified blocks finished"); + // } } pub fn access_log(&self, line_checker: F) -> io::Result diff --git a/test/src/rpc.rs b/test/src/rpc.rs index f5c2127704..28d95aeb6f 100644 --- a/test/src/rpc.rs +++ b/test/src/rpc.rs @@ -7,9 +7,9 @@ use ckb_error::AnyError; use ckb_jsonrpc_types::{ Alert, BannedAddr, Block, BlockEconomicState, BlockFilter, BlockNumber, BlockTemplate, BlockView, Capacity, CellWithStatus, ChainInfo, EpochNumber, EpochView, EstimateCycles, - HeaderView, LocalNode, OutPoint, PoolTxDetailInfo, RawTxPool, RemoteNode, SyncState, Timestamp, - Transaction, TransactionProof, TransactionWithStatusResponse, TxPoolInfo, Uint32, Uint64, - Version, + HeaderView, IndexerTip, LocalNode, OutPoint, PoolTxDetailInfo, RawTxPool, RemoteNode, + SyncState, Timestamp, Transaction, TransactionProof, TransactionWithStatusResponse, TxPoolInfo, + Uint32, Uint64, Version, }; use ckb_types::core::{ BlockNumber as CoreBlockNumber, Capacity as CoreCapacity, EpochNumber as CoreEpochNumber, @@ -128,6 +128,12 @@ impl RpcClient { .expect("rpc call get_current_epoch") } + pub fn get_indexer_tip(&self) -> Option { + self.inner + .get_indexer_tip() + .expect("rpc call get_indexer_tip") + } + pub fn get_epoch_by_number(&self, number: CoreEpochNumber) -> Option { self.inner .get_epoch_by_number(number.into()) @@ -392,4 +398,6 @@ jsonrpc!( pub fn notify_transaction(&self, tx: Transaction) -> H256; pub fn tx_pool_ready(&self) -> bool; pub fn get_pool_tx_detail_info(&self, _hash: H256) -> PoolTxDetailInfo; + + pub fn get_indexer_tip(&self) -> Option; }); diff --git a/test/src/specs/indexer/basic.rs b/test/src/specs/indexer/basic.rs new file mode 100644 index 0000000000..bd73a848db --- /dev/null +++ b/test/src/specs/indexer/basic.rs @@ -0,0 +1,280 @@ +use crate::node::{connect_all, disconnect_all, waiting_for_sync}; +use crate::util::mining::out_ibd_mode; +use crate::utils::find_available_port; +use crate::{Node, Spec}; +use ckb_logger::{info, warn}; +use ckb_types::packed; +use postgresql_embedded::{Settings, blocking::PostgreSQL}; +use std::cell::RefCell; +use std::thread::sleep; +use std::time::Duration; + +/// Test case to reproduce the rich-indexer uncle block reconstruction bug +/// +/// This test specifically triggers the scenario where: +/// 1. Multiple competing blocks at the same height create uncle blocks +/// 2. Rich-indexer attempts to reconstruct uncle blocks as full blocks +/// 3. The reconstruction creates phantom blocks with corrupted data +/// 4. This causes an infinite rollback/append loop in the synchronizer +#[derive(Default)] +pub struct RichIndexerUncleBlockBug { + postgresql: RefCell>, +} + +impl Spec for RichIndexerUncleBlockBug { + fn before_run(&self) -> Vec { + info!("RichIndexerUncleBlockBug: Initializing test environment"); + + // Create 3 nodes to maximize uncle block creation + let node0 = Node::new(self.name(), "node0"); + let node1 = Node::new(self.name(), "node1"); + let node2 = Node::new(self.name(), "node1"); + let mut nodes = [node0, node1, node2]; + + // Setup embedded PostgreSQL with detailed logging + info!("Setting up PostgreSQL with detailed logging for rich-indexer"); + let postgres_port = 8889; + let mut settings = Settings::default(); + settings.port = postgres_port; + settings.host = "127.0.0.1".to_string(); + settings.temporary = true; + settings.username = "postgres".to_string(); + settings.password = "postgres".to_string(); + settings.timeout = Some(Duration::from_secs(60)); + + // Enable detailed PostgreSQL logging to capture the bug + let configs = [ + ("log_directory", "/tmp/postgres_uncle_bug"), + ("log_filename", "bug.log"), + ("logging_collector", "on"), + ("log_statement", "all"), + ("log_min_duration_statement", "0"), + ("shared_preload_libraries", "auto_explain"), + ]; + + for (key, value) in configs { + settings.configuration.insert(key.into(), value.into()); + } + + let mut postgresql = PostgreSQL::new(settings.clone()); + postgresql.setup().expect("Failed to setup PostgreSQL"); + postgresql.start().expect("Failed to start PostgreSQL"); + postgresql + .create_database("ckb_rich_indexer_uncle_test") + .unwrap(); + + info!( + "PostgreSQL started on port {} with detailed logging", + postgres_port + ); + *self.postgresql.borrow_mut() = Some(postgresql); + + // Configure node0 with rich-indexer and aggressive settings to trigger the bug + let node0 = &mut nodes[0]; + node0.modify_app_config(|config| { + config + .rpc + .modules + .push(ckb_app_config::RpcModule::RichIndexer); + config.indexer.rich_indexer = ckb_app_config::RichIndexerConfig { + db_type: ckb_app_config::DBDriver::Postgres, + db_host: "127.0.0.1".to_string(), + db_port: postgres_port, + db_user: settings.username.clone(), + db_password: settings.password.clone(), + db_name: "ckb_rich_indexer_uncle_test".to_string(), + ..Default::default() + }; + + // Aggressive settings to increase uncle block probability + config.indexer.poll_interval = 1; // Poll every 100ms + config.indexer.index_tx_pool = false; + config.logger.log_to_stdout = true; + config.logger.filter = + Some("info,ckb_indexer_sync=trace,ckb_rich_indexer=trace".to_string()); + + // // Mining settings to create more competing blocks + // config.miner.workers = 1; + }); + + // Configure other nodes for competitive mining + for (i, node) in nodes.iter_mut().enumerate().skip(1) { + node.modify_app_config(|config| { + config.logger.log_to_stdout = false; + // config.miner.workers = 1; + }); + } + + // Start all nodes + nodes.iter_mut().for_each(|node| { + info!("started node"); + node.start(); + }); + + nodes.to_vec() + } + + /// Reproduces the specific uncle block reconstruction bug + /// + /// Attack Plan: + /// 1. Create initial sync between all nodes + /// 2. Disconnect nodes to create isolated mining environments + /// 3. Generate competing blocks at the same height with different transaction sets + /// 4. Reconnect nodes to trigger uncle block processing + /// 5. Monitor rich-indexer for the infinite loop bug + fn run(&self, nodes: &mut Vec) { + info!("=== Phase 1: Initial Setup and Sync ==="); + let node0 = &nodes[0]; + let node1 = &nodes[1]; + let node2 = &nodes[2]; + + // Connect all nodes and establish initial chain + info!("connect all nodes"); + connect_all(nodes); + info!("mine until out bootstrap period"); + node0.mine_until_out_bootstrap_period(); + info!("out ibd mode"); + out_ibd_mode(nodes); + info!("writing for sync"); + waiting_for_sync(nodes); + + let initial_height = node0.get_tip_block_number(); + info!("All nodes synced to height {}", initial_height); + + let print_indexer_tip = |node: &Node| -> String { + let indexer_tip = node.rpc_client().get_indexer_tip().unwrap(); + let indexer_tip_number: u64 = indexer_tip.block_number.into(); + format!("{}-{}", indexer_tip_number, indexer_tip.block_hash) + }; + + ////////////////////////////////////////////////////////////////////////////////// + + info!("\n\n\n\n------------ begin"); + let block_13 = node1.new_block_builder(None, None, None).build(); + let uncle_13 = block_13 + .as_advanced_builder() + .timestamp(block_13.timestamp() + 1) + .build(); + + info!( + "=========== constructed :\nblock:{}-{}\nuncle:{}-{}\n", + block_13.number(), + block_13.hash(), + uncle_13.number(), + uncle_13.hash(), + ); + node0.process_block_without_verify(&uncle_13, false); + node0.process_block_without_verify(&block_13, false); + + node1.process_block_without_verify(&block_13, false); + node2.process_block_without_verify(&block_13, false); + node1.process_block_without_verify(&uncle_13, false); + node2.process_block_without_verify(&uncle_13, false); + + info!("\n\n"); + { + let block_14 = node1.new_block_builder(None, None, None).build(); + let uncle_14 = block_14 + .as_advanced_builder() + .timestamp(block_14.timestamp() + 1) + .build(); + + node1.process_block_without_verify(&uncle_14, false); + node0.connect(&node1); + waiting_for_sync(&[node0, node1]); + + node2.process_block_without_verify(&block_14, false); + + info!( + "node0 tip: {}-{}, parent:{}", + node0.get_tip_block_number(), + node0.get_tip_block().hash(), + node0.get_tip_block().parent_hash() + ); + info!( + "node1 tip: {}-{}, parent:{}", + node1.get_tip_block_number(), + node1.get_tip_block().hash(), + node1.get_tip_block().parent_hash() + ); + info!( + "node2 tip: {}-{}, parent:{}", + node2.get_tip_block_number(), + node2.get_tip_block().hash(), + node2.get_tip_block().parent_hash() + ); + + connect_all(nodes); + node2.mine(1); + waiting_for_sync(nodes); + node2.mine(2); + waiting_for_sync(nodes); + } + + let now = std::time::Instant::now(); + while now.elapsed().lt(&Duration::from_secs(60)) { + let tip = node0.get_tip_block(); + let indexer_tip = node0.rpc_client().get_indexer_tip().unwrap(); + let indexer_tip_number: u64 = indexer_tip.block_number.into(); + info!( + "node0 tip: {}-{}, indexer-tip:{}-{}", + tip.number(), + tip.hash(), + indexer_tip_number, + indexer_tip.block_hash + ); + if tip.hash() == indexer_tip.block_hash.into() { + break; + } + sleep(Duration::from_secs(3)); + } + + // { + // let (block, uncle) = node1.construct_uncle(); + // info!( + // "=========== constructed :\nblock:{}-{}\nuncle:{}-{}\n ", + // block.number(), + // block.hash(), + // uncle.number(), + // uncle.hash(), + // ); + + // node1.process_block_without_verify(&uncle, false); + // connect_all(nodes); + // waiting_for_sync(nodes); + // sleep(Duration::from_secs(15)); + // info!("node0 indexer tip: {}", print_indexer_tip(node0)); + + // node1.process_block_without_verify(&block, false); + + // node2.process_block_without_verify(&block, false); + // node2.process_block_without_verify(&uncle, false); + // node2.mine(1); + // connect_all(nodes); + // } + + // waiting_for_sync(nodes); + // sleep(Duration::from_secs(15)); + // info!("node0 indexer tip: {}", print_indexer_tip(node0)); + // { + // info!("checking node0's tip and indexer tip"); + // let tip = node0.get_tip_block(); + // info!( + // "node0 tip: {}-{}, indexer-tip:{}", + // tip.number(), + // tip.hash(), + // print_indexer_tip(node0) + // ); + // } + } + + fn modify_app_config(&self, config: &mut ckb_app_config::CKBAppConfig) { + // Disable automatic peer discovery to control test environment + config.network.connect_outbound_interval_secs = 100_000; + config.network.discovery_local_address = false; + + // Aggressive mining settings to increase competition + // config.tx_pool.min_fee_rate = 0.into(); + // config.tx_pool.max_tx_pool_size = 1000; + } +} diff --git a/test/src/specs/indexer/mod.rs b/test/src/specs/indexer/mod.rs new file mode 100644 index 0000000000..af9382442a --- /dev/null +++ b/test/src/specs/indexer/mod.rs @@ -0,0 +1,3 @@ +mod basic; + +pub use basic::RichIndexerUncleBlockBug; diff --git a/test/src/specs/mod.rs b/test/src/specs/mod.rs index d981a242a2..866a3f41ca 100644 --- a/test/src/specs/mod.rs +++ b/test/src/specs/mod.rs @@ -9,6 +9,7 @@ mod relay; mod rpc; mod sync; mod tx_pool; +mod indexer; pub use alert::*; pub use consensus::*; @@ -21,6 +22,7 @@ pub use relay::*; pub use rpc::*; pub use sync::*; pub use tx_pool::*; +pub use indexer::*; use crate::Node; use ckb_app_config::CKBAppConfig; diff --git a/test/src/utils.rs b/test/src/utils.rs index 9084dbe37c..aa7e07523d 100644 --- a/test/src/utils.rs +++ b/test/src/utils.rs @@ -133,7 +133,7 @@ where if f() { return true; } - thread::sleep(Duration::new(1, 0)); + thread::sleep(Duration::from_millis(100)); } false } diff --git a/test/template/ckb.toml b/test/template/ckb.toml index 4cf431ec87..f34c063367 100644 --- a/test/template/ckb.toml +++ b/test/template/ckb.toml @@ -10,7 +10,7 @@ data_dir = "data" spec = { file = "specs/integration.toml" } [logger] -filter = "info,ckb-rpc=debug,ckb-sync=debug,ckb-relay=debug,ckb-tx-pool=debug,ckb-network=debug" +filter = "info,index=trace,sqlx=debug,ckb-tx-pool=error" color = true log_to_file = true log_to_stdout = false diff --git a/util/app-config/src/app_config.rs b/util/app-config/src/app_config.rs index bbc283852c..1ed9dd5b05 100644 --- a/util/app-config/src/app_config.rs +++ b/util/app-config/src/app_config.rs @@ -35,8 +35,7 @@ pub enum AppConfig { /// directory. /// /// **Attention:** Changing the order of fields will break integration test, see module doc. -#[derive(Clone, Debug, Serialize)] -#[serde(deny_unknown_fields)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct CKBAppConfig { /// The binary name. #[serde(skip)] @@ -263,14 +262,16 @@ impl AppConfig { impl CKBAppConfig { /// Load a new instance from a file pub fn load_from_slice(slice: &[u8]) -> Result { - let legacy_config: legacy::CKBAppConfig = toml::from_slice(slice)?; - for field in legacy_config.deprecated_fields() { - eprintln!( - "WARN: the option \"{}\" in configuration files is deprecated since v{}.", - field.path, field.since - ); - } - Ok(legacy_config.into()) + let config = toml::from_slice(slice)?; + Ok(config) + // let legacy_config: legacy::CKBAppConfig = toml::from_slice(slice)?; + // for field in legacy_config.deprecated_fields() { + // eprintln!( + // "WARN: the option \"{}\" in configuration files is deprecated since v{}.", + // field.path, field.since + // ); + // } + // Ok(legacy_config.into()) } fn derive_options(mut self, root_dir: &Path, subcommand_name: &str) -> Result { diff --git a/util/app-config/src/configs/store.rs b/util/app-config/src/configs/store.rs index 054894b59b..9ce925b161 100644 --- a/util/app-config/src/configs/store.rs +++ b/util/app-config/src/configs/store.rs @@ -1,8 +1,8 @@ -use serde::Serialize; +use serde::{Deserialize, Serialize}; // The default values are set in the legacy version. /// Store config options. -#[derive(Copy, Clone, Serialize, Eq, PartialEq, Hash, Debug)] +#[derive(Copy, Clone, Serialize, Eq, PartialEq, Hash, Debug, Deserialize)] pub struct Config { /// The maximum number of cached block headers. pub header_cache_size: usize, @@ -15,7 +15,9 @@ pub struct Config { /// The maximum number of blocks which uncles section is cached. pub block_uncles_cache_size: usize, /// The maximum number of blocks which extension section is cached. + #[serde(default)] pub block_extensions_cache_size: usize, /// whether enable freezer + #[serde(default)] pub freezer_enable: bool, } diff --git a/util/app-config/src/configs/tx_pool.rs b/util/app-config/src/configs/tx_pool.rs index 5175ac09b1..725b0ad6a3 100644 --- a/util/app-config/src/configs/tx_pool.rs +++ b/util/app-config/src/configs/tx_pool.rs @@ -7,7 +7,7 @@ use url::Url; // The default values are set in the legacy version. /// Transaction pool configuration -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct TxPoolConfig { /// Keep the transaction pool below mb pub max_tx_pool_size: usize, @@ -22,8 +22,10 @@ pub struct TxPoolConfig { /// max ancestors size limit for a single tx pub max_ancestors_count: usize, /// rejected tx time to live by days + #[serde(default)] pub keep_rejected_tx_hashes_days: u8, /// rejected tx count limit + #[serde(default)] pub keep_rejected_tx_hashes_count: u64, /// The file to persist the tx pool on the disk when tx pool have been shutdown. /// @@ -36,6 +38,7 @@ pub struct TxPoolConfig { #[serde(default)] pub recent_reject: PathBuf, /// The expiration time for pool transactions in hours + #[serde(default)] pub expiry_hours: u8, } diff --git a/util/app-config/src/legacy/store.rs b/util/app-config/src/legacy/store.rs index c1cbbed0d3..2667c7e00c 100644 --- a/util/app-config/src/legacy/store.rs +++ b/util/app-config/src/legacy/store.rs @@ -1,7 +1,6 @@ use serde::Deserialize; #[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] pub(crate) struct StoreConfig { header_cache_size: usize, cell_data_cache_size: usize, diff --git a/util/app-config/src/legacy/tx_pool.rs b/util/app-config/src/legacy/tx_pool.rs index 214aaffa70..821e0044ee 100644 --- a/util/app-config/src/legacy/tx_pool.rs +++ b/util/app-config/src/legacy/tx_pool.rs @@ -19,7 +19,7 @@ const DEFAULT_EXPIRY_HOURS: u8 = 12; const DEFAULT_MAX_TX_POOL_SIZE: usize = 180_000_000; #[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] +// #[serde(deny_unknown_fields)] #[allow(dead_code)] pub(crate) struct TxPoolConfig { #[serde(default = "default_max_tx_pool_size")] diff --git a/util/jsonrpc-types/src/indexer.rs b/util/jsonrpc-types/src/indexer.rs index 0f6099e5a1..64e0b73ba5 100644 --- a/util/jsonrpc-types/src/indexer.rs +++ b/util/jsonrpc-types/src/indexer.rs @@ -4,7 +4,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// Indexer tip information -#[derive(Serialize, JsonSchema)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] pub struct IndexerTip { /// indexed tip block hash pub block_hash: H256, diff --git a/util/launcher/src/lib.rs b/util/launcher/src/lib.rs index 4fe120ef20..61cb8ac7b8 100644 --- a/util/launcher/src/lib.rs +++ b/util/launcher/src/lib.rs @@ -248,6 +248,18 @@ impl Launcher { /// Check indexer config pub fn check_indexer_config(&self) -> Result<(), ExitCode> { // check if indexer and rich-indexer are both set + { + // read ckb.toml: self.args.config.root_dir + ckb.toml + let ckb_toml_file = self.args.config.root_dir.join("ckb.toml"); + let content = std::fs::read_to_string(&ckb_toml_file).map_err(|err| { + eprintln!("Read ckb.toml error: {err:?}"); + ExitCode::Config + })?; + info!( + "ckb.toml content: \n{},\nself, args, cofnig:{:?}", + content, self.args.config + ); + } if (self.args.indexer || self.args.config.rpc.indexer_enable()) && (self.args.rich_indexer || self.args.config.rpc.rich_indexer_enable()) { diff --git a/util/rich-indexer/src/indexer/insert.rs b/util/rich-indexer/src/indexer/insert.rs index bf76bf71d5..a54208386b 100644 --- a/util/rich-indexer/src/indexer/insert.rs +++ b/util/rich-indexer/src/indexer/insert.rs @@ -10,6 +10,7 @@ use ckb_types::{ packed::{Byte, CellInput, CellOutput, OutPoint, ScriptBuilder}, prelude::*, }; +use log::info; use sql_builder::SqlBuilder; use sqlx::{ Row, Transaction, @@ -98,15 +99,29 @@ pub(crate) async fn insert_uncle_blocks( block_view: &BlockView, tx: &mut Transaction<'_, Any>, ) -> Result, Error> { + info!( + "insert_ucnle_blocks for block:{}-{},uncles:{}", + block_view.number(), + block_view.hash(), + block_view.uncles().into_iter().len() + ); let uncle_blocks = block_view .uncles() .into_iter() .map(|uncle| { let uncle_block_header = uncle.header(); - BlockView::new_advanced_builder() + let constructed_block = BlockView::new_advanced_builder() .header(uncle_block_header) .proposals(uncle.data().proposals()) - .build() + .build(); + info!( + "Indexer constructed BlockView({}-{}) from UncleView({}-{})", + constructed_block.number(), + constructed_block.hash(), + uncle.number(), + uncle.hash(), + ); + constructed_block }) .collect::>(); let uncle_block_rows: Vec> = uncle_blocks diff --git a/util/rich-indexer/src/service.rs b/util/rich-indexer/src/service.rs index 5996f5528e..14eba9ddfe 100644 --- a/util/rich-indexer/src/service.rs +++ b/util/rich-indexer/src/service.rs @@ -31,9 +31,10 @@ impl RichIndexerService { async_handle: Handle, ) -> Self { let mut store = SQLXPool::default(); + log::info!("RichIndexer config: {:?}", &config.rich_indexer); async_handle .block_on(store.connect(&config.rich_indexer)) - .expect("Failed to connect to rich-indexer database"); + .unwrap_or_else(|err| panic!("Failed to connect to rich-indexer database:{:?}", err)); let sync = IndexerSyncService::new( ckb_db, diff --git a/util/rich-indexer/src/store.rs b/util/rich-indexer/src/store.rs index 237912afe9..3f8758641d 100644 --- a/util/rich-indexer/src/store.rs +++ b/util/rich-indexer/src/store.rs @@ -2,7 +2,7 @@ use anyhow::{Result, anyhow}; use ckb_app_config::{DBDriver, RichIndexerConfig}; use futures::TryStreamExt; use include_dir::{Dir, include_dir}; -use log::LevelFilter; +use log::{LevelFilter, info}; use sqlx::{ AnyPool, ConnectOptions, IntoArguments, Row, Transaction, any::{Any, AnyArguments, AnyConnectOptions, AnyPoolOptions, AnyRow}, @@ -47,6 +47,9 @@ impl SQLXPool { .max_connections(10) .min_connections(0) .acquire_timeout(Duration::from_secs(60)) + .acquire_time_level(LevelFilter::Trace) + .acquire_slow_level(LevelFilter::Trace) + .acquire_slow_threshold(std::time::Duration::from_secs(0)) .max_lifetime(Duration::from_secs(1800)) .idle_timeout(Duration::from_secs(30)); if db_config.store == Into::::into(MEMORY_DB) { @@ -72,8 +75,9 @@ impl SQLXPool { DBDriver::Postgres => { self.postgres_init(db_config).await?; let uri = build_url_for_postgres(db_config); - let connection_options = - AnyConnectOptions::from_str(&uri)?.log_statements(LevelFilter::Trace); + let connection_options = AnyConnectOptions::from_str(&uri)? + .log_statements(LevelFilter::Trace) + .log_slow_statements(LevelFilter::Trace, std::time::Duration::from_secs(0)); let pool = pool_options.connect_with(connection_options).await?; log::info!("PostgreSQL is connected."); self.pool @@ -216,11 +220,16 @@ impl SQLXPool { let tmp_pool_options = AnyPoolOptions::new(); let pool = tmp_pool_options.connect_with(connection_options).await?; // Check if database exists + info!( + "!!!!!!!!!!! !!!!!!!!!!! Checking if database '{}' exists...", + db_config.db_name + ); let query = SQLXPool::new_query(r#"SELECT EXISTS (SELECT FROM pg_database WHERE datname = $1)"#) .bind(db_config.db_name.as_str()); let row = query.fetch_one(&pool).await?; // If database does not exist, create it + info!("!!!!!!!!!!!!!! checking exist done"); if !row.get::(0) { let query = format!(r#"CREATE DATABASE "{}""#, db_config.db_name); SQLXPool::new_query(&query).execute(&pool).await?;