diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 3f51d3fbb9..73205dc120 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -283,7 +283,13 @@ jobs: run: make check-ci - name: Run native checks run: make rust-check + - name: Prepare Docker worker tests + run: | + docker info > /dev/null + docker pull alpine:3.23 - name: Run unit and integration tests + env: + SQLBUILD_REQUIRE_DOCKER_TESTS: "1" run: make test e2e_shards: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f0ab39173e..ba45c64ca6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,19 @@ For a change confined to one Rust crate, run its focused tests before rebuilding cargo test -p sqlbuild-sqltext given_model_files_when_matching_headers_then_offsets_follow_the_python_pattern ``` +Control-plane integration tests require Docker and run both SQLite and PostgreSQL: + +```bash +make test-native NATIVE_TEST_ARGS='-p sqlbuild-orchestration --lib' +``` + +The same Testcontainers harness runs under `make rust-check` in CI. Each invocation starts one +digest-pinned PostgreSQL container, waits for database readiness, shares it across the suite with +an isolated schema per fixture, and removes it after success or failure. No host PostgreSQL setup +is needed. Missing Docker, startup failures or missing database configuration fail the gate; +PostgreSQL cases never silently skip. `NATIVE_TEST_ARGS` accepts Cargo test arguments; its default +runs the workspace with all features. + Keep the toolchain, `CARGO_TARGET_DIR`, profile and `RUSTFLAGS` consistent between iterations. Changing them can invalidate compiled artifacts. Coordinate builds that share a target directory; do not delete another task's artifacts or change global compiler-cache settings. The first build diff --git a/Cargo.lock b/Cargo.lock index 5e2d5968a1..37cab0cc6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.5" @@ -11,6 +17,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "ar_archive_writer" version = "0.5.3" @@ -32,6 +53,21 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "attribute-derive" version = "0.10.5" @@ -68,6 +104,87 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bit-set" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bitflags" version = "2.13.1" @@ -106,6 +223,30 @@ dependencies = [ "objc2", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "bstr" version = "1.13.1" @@ -117,6 +258,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "castaway" version = "0.2.4" @@ -148,12 +307,57 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "collection_literals" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" +[[package]] +name = "command-fds" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b60b5124979fccd9addd89d8b97a1d6eebb4950694520c75ddd722535ea443f" +dependencies = [ + "nix 0.31.3", + "thiserror 2.0.21", +] + [[package]] name = "command-group" version = "5.0.1" @@ -184,6 +388,18 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -202,6 +418,41 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "croner" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa42bcd3d846ebf66e15bd528d1087f75d1c6c1c66ebff626178a106353c576" +dependencies = [ + "chrono", + "derive_builder", + "strum", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -221,6 +472,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.23" @@ -245,7 +505,42 @@ checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" dependencies = [ "dispatch2", "nix 0.31.3", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", ] [[package]] @@ -259,6 +554,37 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -267,6 +593,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -281,6 +608,23 @@ dependencies = [ "objc2", ] +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "either" version = "1.18.0" @@ -300,7 +644,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", ] [[package]] @@ -340,12 +705,98 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -397,8 +848,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", ] [[package]] @@ -408,8 +873,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", - "r-efi", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -431,6 +899,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", ] @@ -455,6 +925,278 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.11", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -463,6 +1205,8 @@ checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -480,6 +1224,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is-macro" version = "0.3.8" @@ -507,6 +1257,17 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "junction" version = "2.1.0" @@ -514,7 +1275,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde48f36530cbd91b65f050101e6c056d6eb743cea2a8f0f19beef69c6742ff8" dependencies = [ "scopeguard", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -532,6 +1293,18 @@ dependencies = [ "cc", ] +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.9.5", +] + [[package]] name = "libsqlite3-sys" version = "0.35.0" @@ -549,12 +1322,33 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "manyhow" version = "0.11.4" @@ -578,6 +1372,22 @@ dependencies = [ "quote", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -602,6 +1412,33 @@ dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "nix" version = "0.27.1" @@ -625,6 +1462,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "objc2" version = "0.6.5" @@ -664,13 +1510,57 @@ dependencies = [ "indexmap", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "phf" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", ] [[package]] @@ -680,7 +1570,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", - "phf_shared", + "phf_shared 0.11.3", ] [[package]] @@ -689,8 +1579,8 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", - "rand", + "phf_shared 0.11.3", + "rand 0.8.8", ] [[package]] @@ -702,12 +1592,33 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "polyglot-sql" version = "0.13.9" @@ -744,6 +1655,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -753,6 +1673,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-utils" version = "0.10.0" @@ -773,6 +1702,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "chacha20", + "core_detect", + "num-traits", + "rand 0.10.3", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "psm" version = "0.1.32" @@ -845,11 +1794,73 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.47" +name = "quick-error" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ "proc-macro2", ] @@ -875,6 +1886,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -889,7 +1906,18 @@ checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -899,7 +1927,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -911,6 +1939,30 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rayon" version = "1.12.0" @@ -931,6 +1983,24 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "580c875845b41ecb1564ad1ae4703044cef774145e3d7a94dc2b3fa7c97868fd" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.13.1" @@ -960,6 +2030,58 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 1.0.11", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "ruff_python_ast" version = "0.0.4" @@ -1065,7 +2187,42 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1074,6 +2231,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.23" @@ -1105,6 +2274,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.229" @@ -1149,6 +2324,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -1158,6 +2344,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -1195,17 +2399,42 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] [[package]] name = "sqlbuild-analysis" @@ -1252,7 +2481,7 @@ dependencies = [ "syn 2.0.119", "syn 3.0.3", "thiserror 2.0.21", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1273,9 +2502,14 @@ name = "sqlbuild-cli" version = "0.186.2" dependencies = [ "blake3", + "chrono", + "command-fds", "ctrlc", "getrandom 0.4.3", + "reqwest", + "rusqlite", "rustix", + "same-file", "serde", "serde_json", "sqlbuild-build-features", @@ -1284,8 +2518,11 @@ dependencies = [ "sqlbuild-core", "sqlbuild-cursor", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-rules", "tempfile", + "tokio", + "uuid", ] [[package]] @@ -1343,6 +2580,34 @@ dependencies = [ "sqlbuild-core", ] +[[package]] +name = "sqlbuild-orchestration" +version = "0.186.2" +dependencies = [ + "axum", + "chrono", + "chrono-tz", + "croner", + "flate2", + "getrandom 0.4.3", + "proptest", + "reqwest", + "rusqlite", + "semver", + "serde", + "serde_json", + "sqlbuild-recall", + "sqlbuild-state", + "sqlx-core", + "sqlx-postgres", + "subtle", + "tempfile", + "thiserror 2.0.21", + "tokio", + "utoipa", + "uuid", +] + [[package]] name = "sqlbuild-project" version = "0.186.2" @@ -1367,6 +2632,8 @@ dependencies = [ name = "sqlbuild-python" version = "0.186.2" dependencies = [ + "chrono", + "chrono-tz", "fensu-policy", "mimalloc", "pyo3", @@ -1375,6 +2642,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", + "sha2", "signal-hook", "sqlbuild-analysis", "sqlbuild-attachments", @@ -1386,6 +2654,7 @@ dependencies = [ "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-recall", @@ -1396,6 +2665,8 @@ dependencies = [ "sqlbuild-sqltext", "sqlbuild-state", "stacker", + "tokio", + "uuid", ] [[package]] @@ -1403,13 +2674,16 @@ name = "sqlbuild-recall" version = "0.186.2" dependencies = [ "blake3", + "chrono", "rusqlite", "rustix", + "serde", "serde_json", "sha2", "sqlbuild-config", "sqlbuild-core", "tempfile", + "utoipa", ] [[package]] @@ -1495,6 +2769,87 @@ dependencies = [ "sqlbuild-cursor", ] +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "smallvec", + "thiserror 2.0.21", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.8", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.21", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "stacker" version = "0.1.25" @@ -1505,7 +2860,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1514,6 +2869,50 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -1536,6 +2935,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -1552,7 +2971,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1601,6 +3020,16 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.13.3" @@ -1608,16 +3037,64 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] -name = "toml" -version = "0.9.12+spec-1.1.0" +name = "tokio" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ - "indexmap", - "serde_core", - "serde_spanned", - "toml_datetime 0.7.5+spec-1.1.0", - "toml_parser", + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", "toml_writer", "winnow 0.7.15", ] @@ -1667,12 +3144,107 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1688,6 +3260,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -1706,7 +3284,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" dependencies = [ - "phf", + "phf 0.11.3", "unicode_names2_generator", ] @@ -1719,7 +3297,7 @@ dependencies = [ "getopts", "log", "phf_codegen", - "rand", + "rand 0.8.8", ] [[package]] @@ -1728,6 +3306,76 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] + +[[package]] +name = "utf_types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + +[[package]] +name = "utoipa" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0b99ee52df3028635d93840c797102da61f8a7bb3cf751032455895b52ef8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "uuid", +] + +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "sha1_smol", + "wasm-bindgen", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -1740,6 +3388,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -1750,12 +3407,139 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.3", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.11", +] + +[[package]] +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1778,7 +3562,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1787,12 +3571,83 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1802,6 +3657,127 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.15" @@ -1817,6 +3793,41 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.62" @@ -1837,6 +3848,72 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 6014fffc19..fd65e8af7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/sqlbuild-config", "crates/sqlbuild-host", "crates/sqlbuild-recall", + "crates/sqlbuild-orchestration", "crates/sqlbuild-discovery", "crates/sqlbuild-scopes", "crates/sqlbuild-model-config", @@ -39,6 +40,7 @@ order = [ "sqlbuild-config", "sqlbuild-host", "sqlbuild-recall", + "sqlbuild-orchestration", "sqlbuild-model-config", "sqlbuild-discovery", "sqlbuild-render", @@ -62,6 +64,7 @@ publish = false rust-version = "1.88" [workspace.dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] } fensu-policy = "=0.22.0" # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. getrandom = "0.4" @@ -96,6 +99,7 @@ syn-v3-macro-features = { package = "syn", version = "3", default-features = fal thiserror = "2" # Windows only: `sqb` ignores Ctrl+C while the Python CLI it started handles it. ctrlc = "3" +command-fds = "0.3" quote = "1" rayon = "1" # Static reading of project Python source; the parser fensu uses. @@ -112,13 +116,27 @@ errno = "0.3" windows-sys = { version = "0.61", default-features = false, features = ["Win32_System_Diagnostics_Debug"] } # Recall's local store: SQLite compiled in, so every build has the same engine version. rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } -rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } +rustix = { version = "1.1.4", default-features = false, features = ["fs", "process", "std", "termios"] } saphyr-parser = "0.1" regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } serde = { version = "1", features = ["derive"] } # Every crate relies on insertion-ordered JSON objects, which the extension has always built with. serde_json = { version = "1", features = ["preserve_order"] } sha2 = "0.10" +chrono = { version = "0.4", features = ["serde"] } +chrono-tz = "0.10" +croner = "=3.0.1" +flate2 = "1" +proptest = "1" +same-file = "1" +semver = "1" +# Pin SQLx's driver crates together: its umbrella also resolves an incompatible SQLite engine. +sqlx-core = { version = "=0.8.6", default-features = false, features = ["_rt-tokio", "_tls-rustls-ring-webpki", "chrono", "uuid", "json"] } +sqlx-postgres = { version = "=0.8.6", default-features = false, features = ["chrono", "uuid", "json"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time", "signal", "process", "net"] } +uuid = { version = "=1.18.1", features = ["v5", "v7", "serde"] } +utoipa = { version = "5", features = ["chrono", "uuid"] } sqlbuild-analysis = { path = "crates/sqlbuild-analysis" } sqlbuild-attachments = { path = "crates/sqlbuild-attachments" } sqlbuild-build-features = { path = "crates/sqlbuild-build-features" } @@ -133,6 +151,7 @@ sqlbuild-render = { path = "crates/sqlbuild-render" } sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } +sqlbuild-orchestration = { path = "crates/sqlbuild-orchestration" } sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } @@ -141,6 +160,7 @@ sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } +subtle = "2" tempfile = "3" toml = "0.9" toml_edit = { version = "0.25", default-features = false, features = ["parse"] } diff --git a/Makefile b/Makefile index 1753e39f05..911e79832f 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,13 @@ fast-native: rust-check: rust-lint uv run fensu check - cargo test --workspace --all-features + $(MAKE) test-native + +NATIVE_TEST_ARGS ?= --workspace --all-features + +.PHONY: test-native +test-native: + uv run --no-sync python -m scripts.run_native_tests $(NATIVE_TEST_ARGS) .PHONY: rust-lint rust-lint: diff --git a/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql b/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql new file mode 100644 index 0000000000..2d882a92ae --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql @@ -0,0 +1,16 @@ +MODEL ( + description 'Backfill orders over an explicit integer window.', + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor batch_id, + cursor_type integer, + cursor_start 0, + cursor_watermark_mode all, + cursor_inputs (raw_orders (column batch_id, roles [filter, watermark])), + batch_size '2', + batch_concurrency 1, +); +SELECT amount, batch_id FROM __source("raw_orders") +WHERE batch_id >= __cursor_start() AND batch_id < __cursor_end() diff --git a/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql b/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql new file mode 100644 index 0000000000..fa8b2bcd25 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql @@ -0,0 +1,17 @@ +MODEL ( + description 'Backfill orders over an explicit timestamp window.', + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor ordered_at, + cursor_type timestamp, + cursor_grain day, + cursor_start '2026-07-01', + cursor_watermark_mode all, + cursor_inputs (raw_orders (column ordered_at, roles [filter, watermark])), + batch_size 1d, + batch_concurrency 1, +); +SELECT amount, ordered_at FROM __source("raw_orders") +WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py new file mode 100644 index 0000000000..7beff48f02 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py @@ -0,0 +1,17 @@ +"""Provider-backed fulfillment times evaluated in the code location, never by the scheduler.""" + +from sqlbuild.orchestration import calendar_schedule, job + +orders = job("orders", description="Build order reporting.", select="orders") + + +@calendar_schedule( + job=orders, + description="Refresh orders at upcoming fulfillment times.", + horizon="8s", + timezone="America/New_York", + catch_up="all", + max_catch_up=10, +) +def fulfillment_calendar(window, fulfillment): + return fulfillment.upcoming(window) diff --git a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py new file mode 100644 index 0000000000..ae5a4c9dec --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py @@ -0,0 +1,10 @@ +"""Negative fixture: mutually exclusive build flags must fail before publication.""" + +from sqlbuild.orchestration import job + +orders = job( + "orders", + description="Refresh orders.", + select="orders", + settings={"load_sources": False, "reload": True}, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py new file mode 100644 index 0000000000..f448f2b85f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py @@ -0,0 +1,15 @@ +"""Polling callbacks return durable run keys and keep their cursor in the native transaction.""" + +from sqlbuild.orchestration import job, sensor + +orders = job("annual_orders", description="Build orders after source inspection.", select="orders") + + +@sensor( + "poll_orders", description="Poll fulfillment readiness.", run=orders, minimum_interval_seconds=1 +) +def poll_orders(ctx, fulfillment): + def poll(): + return fulfillment.poll(ctx) + + return poll() diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import.py b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py new file mode 100644 index 0000000000..6483f0b2ab --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py @@ -0,0 +1,3 @@ +"""Negative fixture: a failed declaration import must fail every publication retry.""" + +raise RuntimeError("broken declaration") diff --git a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py new file mode 100644 index 0000000000..248b1b4ce1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py @@ -0,0 +1,10 @@ +"""Negative fixture: job settings must not override the frozen selection.""" + +from sqlbuild.orchestration import job + +orders = job( + "orders", + description="Refresh orders.", + select="orders", + settings={"select": "customers"}, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql new file mode 100644 index 0000000000..fe7ba68f98 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql @@ -0,0 +1,10 @@ +-- SQLite predecessor text can carry the writer's offset, across the UTC date boundary. +-- Load after legacy_warehouse_state.sql, before any writer adds event_id. +INSERT INTO main._sqlbuild_fingerprints ( + node_type, node_name, target_database, target_schema, target_name, run_id, + definition_hash, version_hash, schema_fingerprint, definition_b64, metadata_json_b64, ts +) VALUES ( + 'model', 'offset_orders', NULL, 'main', 'offset_orders', 'offset-run', + 'legacy-definition', 'legacy-version', 'legacy-schema', + 'U0VMRUNUIDEgQVMgb3JkZXJfaWQ=', 'e30=', '2025-12-31T23:00:00-11:00' +); diff --git a/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql new file mode 100644 index 0000000000..b135ecff51 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql @@ -0,0 +1,22 @@ +-- Populated predecessor schemas for additive event-ID and read-only history-sync checks. +CREATE TABLE main._sqlbuild_fingerprints ( + node_type VARCHAR NOT NULL, node_name VARCHAR NOT NULL, + target_database VARCHAR, target_schema VARCHAR, target_name VARCHAR, + run_id VARCHAR NOT NULL, definition_hash VARCHAR NOT NULL, version_hash VARCHAR NOT NULL, + schema_fingerprint VARCHAR NOT NULL, definition_b64 VARCHAR NOT NULL, + metadata_json_b64 VARCHAR NOT NULL, ts TIMESTAMP NOT NULL +); +INSERT INTO main._sqlbuild_fingerprints VALUES ( + 'model', 'orders', NULL, 'main', 'orders', 'legacy-run', 'legacy-definition', + 'legacy-version', 'legacy-schema', 'U0VMRUNUIDEgQVMgb3JkZXJfaWQ=', 'e30=', '2026-01-01 10:00:00' +); +CREATE TABLE main._sqlbuild_source_freshness ( + source_name VARCHAR NOT NULL, target_database VARCHAR, target_schema VARCHAR, + target_name VARCHAR, run_id VARCHAR NOT NULL, strategy VARCHAR NOT NULL, + value_kind VARCHAR NOT NULL, data_version VARCHAR, data_version_hash VARCHAR NOT NULL, + observed_at TIMESTAMP NOT NULL +); +INSERT INTO main._sqlbuild_source_freshness VALUES ( + 'raw.orders', NULL, 'main', 'orders', 'legacy-run', 'max_column', 'integer', + '7', 'legacy-value', '2026-01-01 10:00:00' +); diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql b/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql new file mode 100644 index 0000000000..fe752e60d2 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql @@ -0,0 +1,3 @@ +-- A long-running statement for scheduler crash, adoption and cancellation checks. +-- The supervising test cancels this query; it is not part of benchmark measurements. +SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value) diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py new file mode 100644 index 0000000000..35727b2092 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py @@ -0,0 +1,7 @@ +"""A group can contain jobs, but cannot contain another group.""" + +from sqlbuild.orchestration import job, job_group + +orders = job("orders_job", description="Build orders.", select="orders") +daily = job_group("daily_orders", description="Build daily orders.", jobs=[orders]) +weekly = job_group("weekly_orders", description="Unsupported nested membership.", jobs=[daily]) diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders.py b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py new file mode 100644 index 0000000000..ac8f6cb8fc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py @@ -0,0 +1,12 @@ +"""Trigger on each real orders rebuild, not on freshness-only success.""" + +from sqlbuild.orchestration import NodeEventKind, job, node_event, sensor + +source_orders = job("source_orders", description="Source orders.", select="orders") +inventory = job("inventory", description="Destination inventory.", select="stg_orders") +on_orders = sensor( + "on_orders", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), + run=inventory, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py new file mode 100644 index 0000000000..4814f4ae1e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py @@ -0,0 +1,30 @@ +"""Sensor-only roots retain referenced jobs while target filters and event enums stay native.""" + +from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor + + +def definitions(): + refresh = job("health_orders", description="Refresh orders.", select="orders") + return [ + sensor( + "overdue_orders", + description="Catch up overdue order reporting.", + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + run=refresh, + minimum_interval_seconds=7, + default_status="stopped", + targets=("development",), + ), + sensor( + "other_target_orders", + description="Refresh another target.", + when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), + run=refresh, + targets=("production",), + ), + ] + + +triggers = definitions() +alias = triggers[0] +triggers.append(triggers) diff --git a/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql new file mode 100644 index 0000000000..8304e49843 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql @@ -0,0 +1,8 @@ +MODEL (description 'Retry a failed cast after warehouse input changes.', materialized table); +SELECT CASE + WHEN EXISTS ( + SELECT 1 FROM information_schema.tables WHERE table_name = 'retry_ready' + ) THEN 17 + ELSE CAST(value AS INTEGER) +END AS amount +FROM (VALUES ('invalid')) AS source(value) diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders.py b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py new file mode 100644 index 0000000000..bf6d376187 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py @@ -0,0 +1,16 @@ +"""Freeze source-job selectors without executing user code in the scheduler.""" + +from sqlbuild.orchestration import RunEventKind, job, run_event, sensor + +source_orders = job("source_orders", description="Source orders.", select="orders") +inventory = job("inventory", description="Destination inventory.", select="stg_orders") +on_orders = sensor( + "on_orders", + description="Build inventory only after orders actually succeeds.", + when=run_event( + job=source_orders, + kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders",), + ), + run=inventory, +) diff --git a/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql new file mode 100644 index 0000000000..d49cb02a03 --- /dev/null +++ b/benchmarks/duckshop/project/functions/sql/edge_cases/edge_next_quantity.sql @@ -0,0 +1,8 @@ +FUNCTION ( + description 'Advance an order quantity in the inherited function namespace.', + tags [edge_case], + arguments (quantity INTEGER), + returns INTEGER, +); + +quantity + 1 diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql new file mode 100644 index 0000000000..f86364144c --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/rpt_edge_scheduled_function.sql @@ -0,0 +1,16 @@ +MODEL ( + description 'A scheduled model whose implicit scalar-function writer must be claimed.', + tags [edge_case], + materialized table, + columns ( + quantity (type INTEGER, nullable false, audits [not_null]), + ), +); + +WITH final AS ( + SELECT __udf("edge_next_quantity")(q.quantity) AS quantity + FROM __seed("edge_quantity_steps") AS q +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py new file mode 100644 index 0000000000..78b726a288 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py @@ -0,0 +1,97 @@ +"""Exercise alias deduplication and cyclic factory containers during native discovery.""" + +from sqlbuild.orchestration import ( + PlatformEventKind, + RunEventKind, + job, + job_group, + platform_event, + run_event, + schedule, + sensor, +) + +edge_orders = job( + "edge_snapshot_orders", + description="Refresh only the synthetic edge-case models.", + select="tag:edge_case", + settings={"run_tests": False, "vars": {"edge_delivery_batch": "1"}, "reload": True}, + concurrency=2, + schedule=schedule( + cron="15 2 * * *", + timezone="UTC", + starts_at="2026-07-03T01:17:23.456789+05:30", + default_status="paused", + catch_up="all", + max_catch_up=3, + ), +) +edge_alias = edge_orders +edge_factory_results = [[edge_orders]] +edge_factory_results.append(edge_factory_results) +edge_group = job_group( + "edge_snapshot_group", + description="Exercise group discovery through shared, cyclic declaration containers.", + jobs=[edge_orders], +) +edge_factory_results.append(edge_group) + +edge_inventory = job( + "edge_snapshot_inventory", + description="Exercise a child limit independent of its parent.", + select="tag:edge_case", + concurrency_groups=("edge_products_limit",), +) +edge_products = job( + "edge_snapshot_products", + description="Exercise the opposing parent and child acquisition order.", + select="tag:edge_case", + concurrency_groups=("edge_inventory_limit",), +) +edge_inventory_group = job_group( + "edge_inventory_group", + description="Complete inventory without reserving its child's product limit.", + jobs=[edge_inventory], + concurrency_groups=("edge_inventory_limit",), +) +edge_products_group = job_group( + "edge_products_group", + description="Wait for safe admission when both installation limits are one.", + jobs=[edge_products], + concurrency_groups=("edge_products_limit",), +) +edge_health = sensor( + "edge_platform_orders", + description="Catch up retained platform observations with a stopped group sensor.", + when=platform_event( + kinds=[ + PlatformEventKind.TICK_OVERDUE, + PlatformEventKind.LOCATION_BROKEN, + PlatformEventKind.LAUNCH_FAILED, + ] + ), + run=edge_group, + minimum_interval_seconds=7, + default_status="stopped", +) +edge_factory_results.append(edge_health) + +edge_job_outcomes = sensor( + "edge_job_outcomes", + description="Keep warning and partial outcomes distinct from success.", + when=run_event(job=edge_orders, kinds=[RunEventKind.WARN, RunEventKind.PARTIAL]), + run=edge_inventory, + default_status="stopped", +) +edge_group_outcomes = sensor( + "edge_group_outcomes", + description="Observe group completion including cancellation and presumed failure.", + when=run_event( + job=edge_group, + kinds=[RunEventKind.COMPLETED], + executed_successfully=("tag:edge_case",), + ), + run=edge_inventory, + default_status="stopped", +) +edge_factory_results.extend([edge_job_outcomes, edge_group_outcomes]) diff --git a/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv new file mode 100644 index 0000000000..9e70ca2ea3 --- /dev/null +++ b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.csv @@ -0,0 +1,3 @@ +quantity +2 +7 diff --git a/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml new file mode 100644 index 0000000000..de890aa0b8 --- /dev/null +++ b/benchmarks/duckshop/project/seeds/edge_cases/edge_quantity_steps.yml @@ -0,0 +1,7 @@ +seeds: + - name: edge_quantity_steps + description: Quantities whose seed destination inherits the target namespace. + tags: [edge_case] + columns: + - name: quantity + type: INTEGER diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql new file mode 100644 index 0000000000..8db1011dad --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_rpt_edge_scheduled_function.sql @@ -0,0 +1,15 @@ +TEST (name "rpt_edge_scheduled_function__advances_the_inherited_function_quantity"); + +WITH __seed__edge_quantity_steps AS ( + SELECT v.quantity + FROM VALUES + (-2), + (23) AS v(quantity) +), + +__expected__rpt_edge_scheduled_function AS ( + SELECT v.quantity + FROM VALUES + (-1), + (24) AS v(quantity) +) diff --git a/crates/README.md b/crates/README.md index 37c7b39166..e1727686ae 100644 --- a/crates/README.md +++ b/crates/README.md @@ -107,6 +107,40 @@ the goldens in `tests/goldens/compiler`, with stage captures and full discovery, analysis coverage required. CI also runs it on Python 3.13 and 3.14, and compiles and builds a playground project on Windows. +## Warehouse event identity + +Fingerprint and source-freshness writers persist deterministic `event_id` values in nullable text +columns. Existing tables evolve additively; no predecessor rows are rewritten. Latest-state readers +retain their existing projections. IDs are lowercase SHA-256 hex over compact UTF-8 JSON arrays, +whose fixed namespace is independent of the package version: + +| Table | Array fields, in order | +|---|---| +| `_sqlbuild_fingerprints` | `"sqlbuild.fingerprint.event.v1"`, node kind, node name, target database, target schema, target name, run ID, version hash | +| `_sqlbuild_source_freshness` | `"sqlbuild.source-freshness.event.v1"`, source name, target database, target schema, target name, run ID, observation time, strategy, value kind, value hash | + +Nullable identity fields remain JSON null, distinct from empty strings. Freshness observation time +is normalized to UTC ISO text with six fractional digits and a `+00:00` offset. Retrying the same +observation retains its ID; another run or observation is distinct. Definitions, display values, +and future storage columns do not alter these identities. + +The location history reader pages fingerprint and source-freshness observations without selecting +only the latest row. It is read-only, including local database connections: it neither creates +databases nor evolves tables. Fingerprint definitions and metadata use the existing storage decoder; +raw storage values remain the keyset cursor, separate from decoded UTC timestamps and text. +Source-freshness ingestion preserves warehouse IDs, scoped by location, target and physical state +table. Only missing or NULL IDs use the fixed legacy observation tuple, never a hash of the evolving +full row. Its bounded pages commit atomically under the current location session and snapshot +generation, and repeated pages are idempotent. A history read can hold an +inclusive observation-time lower bound fixed across pages, independently of the row-key cursor. +SQLite's SQL prefilter includes the boundary day and its predecessor so legacy separators, variable +fractional precision and writer offsets crossing a UTC date remain readable; decoded UTC timestamps +enforce the exact bound. Other adapters prefilter at whole-second precision. Continuation follows +the raw row count: even an empty decoded page can have more rows to inspect. Fingerprint ingestion +and per-hash definition storage remain separate from this read-only transport. Automatic polling, +watermark/overlap management and deletion reconciliation are separate location-sync responsibilities, +not implemented by the page reader. + ## Working on the crates - Tests live next to the module they cover; run them for one crate with diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs index 791dcc6822..59e8a2328c 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs @@ -12,8 +12,9 @@ pub const COLUMN_SCHEMA_FINGERPRINT: &str = "schema_fingerprint"; pub const COLUMN_DEFINITION_B64: &str = "definition_b64"; pub const COLUMN_METADATA_JSON_B64: &str = "metadata_json_b64"; pub const COLUMN_TIMESTAMP: &str = "ts"; +pub const COLUMN_EVENT_ID: &str = "event_id"; -/// Every stored column, in table order. +/// Legacy columns used by latest-row readers, including readers of tables without an event ID. pub const FINGERPRINT_COLUMNS: [&str; 12] = [ COLUMN_NODE_TYPE, COLUMN_NODE_NAME, diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs new file mode 100644 index 0000000000..6ef1ccee43 --- /dev/null +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/_event_id.rs @@ -0,0 +1,20 @@ +//! Fixed writer identity excludes timestamps, definitions and future storage columns. + +use sha2::{Digest, Sha256}; + +use crate::fingerprint_storage::models::StoredFingerprint; + +/// Stable SHA-256 of the versioned JSON tuple named by the fingerprint storage contract. +pub(super) fn event_id(row: &StoredFingerprint) -> String { + let identity = serde_json::json!([ + "sqlbuild.fingerprint.event.v1", + row.node_type, + row.node_name, + row.target_database, + row.target_schema, + row.target_name, + row.run_id, + row.version_hash, + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs index 7c0a00587e..db3a126811 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/create_table_sql.rs @@ -1,9 +1,10 @@ //! Fingerprint table DDL. use crate::fingerprint_storage::constants::{ - COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, + COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, + COLUMN_VERSION_HASH, }; /// `CREATE TABLE IF NOT EXISTS` for the fingerprint table, with the adapter's column types. @@ -21,6 +22,7 @@ pub fn create_table_sql(qualified_name: &str, string_type: &str, timestamp_type: {COLUMN_SCHEMA_FINGERPRINT} {string_type} NOT NULL, \ {COLUMN_DEFINITION_B64} {string_type} NOT NULL, \ {COLUMN_METADATA_JSON_B64} {string_type} NOT NULL, \ - {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL)" + {COLUMN_TIMESTAMP} {timestamp_type} NOT NULL, \ + {COLUMN_EVENT_ID} {string_type})" ) } diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs index 25e54bf696..dbd785c797 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/insert_sql.rs @@ -3,7 +3,9 @@ use crate::fingerprint_storage::_helpers::base64::encode; use crate::fingerprint_storage::_helpers::literals::{optional_string_literal, string_literal}; use crate::fingerprint_storage::_helpers::table_columns::column_list; +use crate::fingerprint_storage::constants::COLUMN_EVENT_ID; use crate::fingerprint_storage::errors::FingerprintStorageError; +use crate::fingerprint_storage::main::_event_id::event_id; use crate::fingerprint_storage::models::StoredFingerprint; /// One multi-row `INSERT` for a non-empty batch. @@ -16,14 +18,14 @@ pub fn insert_sql( } let rows: Vec = fingerprints.iter().map(row_values).collect(); Ok(format!( - "INSERT INTO {qualified_name} ({}) VALUES {}", + "INSERT INTO {qualified_name} ({}, {COLUMN_EVENT_ID}) VALUES {}", column_list(), rows.join(", ") )) } fn row_values(fingerprint: &StoredFingerprint) -> String { - let values: [String; 12] = [ + let values: [String; 13] = [ string_literal(&fingerprint.node_type), string_literal(&fingerprint.node_name), optional_string_literal(fingerprint.target_database.as_deref()), @@ -36,6 +38,7 @@ fn row_values(fingerprint: &StoredFingerprint) -> String { format!("'{}'", encode(fingerprint.definition.as_bytes())), format!("'{}'", encode(fingerprint.metadata_json.as_bytes())), format!("'{}'", fingerprint.ts), + string_literal(&event_id(fingerprint)), ]; format!("({})", values.join(", ")) } diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs index 54c1f22e6f..ab782386e7 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/main/mod.rs @@ -1,3 +1,4 @@ +mod _event_id; pub mod create_table_sql; pub mod decode_storage; pub mod insert_batch_lengths; diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs index 59c72c1dd6..5595b45108 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/tests/test_storage.rs @@ -142,8 +142,9 @@ fn given_rows_when_inserting_then_literals_are_escaped_and_definitions_encoded() expected_sql: Ok( "INSERT INTO main.t (node_type, node_name, target_database, target_schema, \ target_name, run_id, definition_hash, version_hash, schema_fingerprint, \ - definition_b64, metadata_json_b64, ts) VALUES ('model', 'o''rders', NULL, \ - 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00')", + definition_b64, metadata_json_b64, ts, event_id) VALUES ('model', 'o''rders', NULL, \ + 'main', 'orders', 'run', 'd', '', 's', 'YWI=', 'e30=', '2026-01-01T00:00:00', \ + 'b9397954c803e7f99f6a233c2985be772c7ddf2c6f234197d77bc227ae175cbd')", ), }, InsertTestCase { diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 4a2253fb56..cac9dd214d 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -17,7 +17,9 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true +chrono.workspace = true getrandom.workspace = true +reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["arbitrary_precision"] } sqlbuild-build-features.workspace = true @@ -26,15 +28,23 @@ sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-cursor.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true sqlbuild-rules.workspace = true +tokio.workspace = true +uuid.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true +[target.'cfg(target_os = "linux")'.dependencies] +command-fds.workspace = true + [target.'cfg(windows)'.dependencies] ctrlc.workspace = true +same-file.workspace = true [dev-dependencies] +rusqlite.workspace = true tempfile.workspace = true [lints] diff --git a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs new file mode 100644 index 0000000000..19bb1563e3 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs @@ -0,0 +1,128 @@ +//! Render one authored setting and check emitted options against the build parser. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::argv::_helpers::table::root_parser; +use crate::argv::constants::CONCURRENCY_DEST; +use crate::argv::models::{ActionKind, ActionSpec, Default, ParserSpec, Value}; + +pub(crate) fn setting_argument( + parser: &ParserSpec, + name: &str, + value: &Json, +) -> Result, String> { + if matches!( + name, + "help" + | "target" + | "project_dir" + | "select" + | "exclude" + | "select_file" + | "positional_select" + | "json" + | "json_output" + | "event_output" + | "manifest" + | "dbt_project_dir" + | "dbt_profiles_dir" + | "dbt_target" + | "dbt_target_path" + ) { + return Err(format!("job setting {name} is owned by the run request")); + } + let actions: Vec<&ActionSpec> = parser + .actions + .iter() + .filter(|action| action.dest.as_deref() == Some(name)) + .collect(); + let Some(action) = actions.first() else { + return Err(format!("unknown build setting {name}")); + }; + if matches!(action.kind, ActionKind::StoreTrue | ActionKind::StoreFalse) { + return boolean_argument(name, value, &actions); + } + let option = &action.option_strings[0]; + let text = setting_text(name, value, action)?; + if action + .choices + .as_ref() + .is_some_and(|choices| !choices.contains(&text)) + { + return Err(format!("job setting {name} is not an accepted choice")); + } + Ok(Some(format!("{option}={text}"))) +} + +fn boolean_argument( + name: &str, + value: &Json, + actions: &[&ActionSpec], +) -> Result, String> { + let action = actions[0]; + let flag = value + .as_bool() + .ok_or_else(|| format!("job setting {name} must be boolean"))?; + if let Some(chosen) = actions + .iter() + .find(|action| action.constant == Value::Bool(flag)) + { + return Ok(Some(chosen.option_strings[0].clone())); + } + if action.default != Default::Value(Value::Bool(flag)) + && !(action.default == Default::Suppress + && root_parser().actions.iter().any(|global| { + global.dest.as_deref() == Some(name) + && global.default == Default::Value(Value::Bool(flag)) + })) + { + return Err(format!("job setting {name} cannot be set to {flag}")); + } + Ok(None) +} + +fn setting_text(name: &str, value: &Json, action: &ActionSpec) -> Result { + match action.converter.as_deref() { + Some("int" | "_positive_integer") => { + let integer = value + .as_i64() + .ok_or_else(|| format!("job setting {name} must be an integer"))?; + if integer < 1 + && (name == CONCURRENCY_DEST + || action.converter.as_deref() == Some("_positive_integer")) + { + return Err(format!("job setting {name} must be positive")); + } + Ok(integer.to_string()) + } + Some("parse_cli_vars") => { + if !value.is_object() { + return Err(format!("job setting {name} must be a JSON object")); + } + Ok(value.to_string()) + } + _ => value + .as_str() + .ok_or_else(|| format!("job setting {name} must be text")) + .map(str::to_owned), + } +} + +pub(crate) fn validate_mutex(parser: &ParserSpec, arguments: &[String]) -> Result<(), String> { + let mut mutex_groups: BTreeMap = BTreeMap::new(); + for argument in arguments { + let option = argument + .split_once('=') + .map_or(argument.as_str(), |(option, _)| option); + if let Some(group) = parser.actions[parser.option_index[option]].mutex_group + && let Some(previous) = mutex_groups.insert(group, option) + { + return Err(format!( + "job settings {previous} and {option} are mutually exclusive" + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs index 22763ff51e..1d2d932e20 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod dbt_execution; pub(crate) mod formatting; +pub(crate) mod job_settings; pub(crate) mod namespace; pub(crate) mod normalization; pub(crate) mod parsing; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs index 311d9ef353..b53bf9bf04 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/normalization.rs @@ -77,24 +77,10 @@ impl Run<'_> { .map_err(|message| self.error(parser, &message))?; namespace.set("concurrency", concurrency); } - if CURSOR_COMMANDS.contains(&command) { - let untyped: bool = ["start_cursor", "end_cursor"] - .iter() - .any(|name| namespace.value(name) != Value::None); - let typed: bool = [ - "start_cursor_ts", - "end_cursor_ts", - "start_cursor_int", - "end_cursor_int", - ] - .iter() - .any(|name| namespace.value(name) != Value::None); - if untyped && typed { - return Err(self.error( - parser, - "--start-cursor/--end-cursor cannot be combined with the -ts or -int cursor flags", - )); - } + if CURSOR_COMMANDS.contains(&command) + && let Some(message) = cursor_error(|name| namespace.value(name) != Value::None) + { + return Err(self.error(parser, message)); } if command == RENAME_COMMAND && let Some(message) = rename_error(&namespace) @@ -156,6 +142,24 @@ impl Run<'_> { } } +/// The command line and authored job settings share the same cursor-family restriction. +pub(crate) fn cursor_error(present: impl Fn(&str) -> bool) -> Option<&'static str> { + let untyped = ["start_cursor", "end_cursor"] + .iter() + .any(|name| present(name)); + let typed = [ + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", + ] + .iter() + .any(|name| present(name)); + (untyped && typed).then_some( + "--start-cursor/--end-cursor cannot be combined with the -ts or -int cursor flags", + ) +} + /// The first rule `--since` and `--explain` break, on the commands that accept them. fn since_error(namespace: &Namespace) -> Option<&'static str> { match namespace.value("since") { diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index d2b8b61be1..038ceefb5b 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -78,6 +78,996 @@ "nargs": "A...", "group": 0, "parsers": [ + { + "name": "location", + "parser": { + "prog": "sqb location", + "description": "Compile and publish a project from its own Python environment. Uses SQLBUILD_LOCATION_TOKEN for scoped HTTP access, never the control database. Results are JSON; progress goes to stderr.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "location_command", + "required": true, + "choices": ["publish", "serve", "runtime", "execute"], + "help": "publish one snapshot, watch and reload, or run the supervisor's stdio compiler", + "group": 0 + }, + { + "kind": "store", + "options": ["--server"], + "dest": "location_server_url", + "required": true, + "help": "control API HTTP or HTTPS origin, without credentials", + "group": 1 + }, + { + "kind": "store", + "options": ["--location-id"], + "dest": "location_id", + "required": true, + "help": "registered location UUID", + "group": 1 + }, + { + "kind": "store", + "options": ["--target"], + "dest": "target", + "required": true, + "help": "configured project target to compile", + "group": 1 + }, + { + "kind": "store", + "options": ["--generation"], + "dest": "location_generation", + "type": "int", + "help": "required for publish; serve derives the next generation from the server", + "group": 1 + }, + { + "kind": "store", + "options": ["--image-id"], + "dest": "location_image_id", + "help": "immutable deployment image identity to pin for workers", + "group": 1 + }, + { + "kind": "store", + "options": ["--run-id"], + "dest": "location_run_id", + "help": "frozen run UUID for execute; requires SQLBUILD_RUN_TOKEN", + "group": 1 + }, + { + "kind": "store", + "options": ["--claim-id"], + "dest": "location_claim_id", + "help": "active claim UUID for execute", + "group": 1 + }, + { + "kind": "store", + "options": ["--project-dir", "--sqb-project-dir"], + "dest": "project_dir", + "default": {"suppress": true}, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + } + ] + } + }, + { + "name": "admin", + "parser": { + "prog": "sqb admin", + "description": "Privileged local administration using direct control database access. Results are JSON; issued bearer credentials appear only once on stdout.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "parsers", + "dest": "admin_command", + "nargs": "A...", + "required": true, + "group": 0, + "parsers": [ + { + "name": "create-location", + "parser": { + "prog": "sqb admin create-location", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "NAME", + "required": true, + "help": "location label; identity is assigned by the control store", + "group": 0 + } + ] + } + }, + { + "name": "create-principal", + "parser": { + "prog": "sqb admin create-principal", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "NAME", + "required": true, + "help": "principal label; creation grants no permissions", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--kind" + ], + "dest": "admin_principal_kind", + "choices": [ + "person", + "service_account" + ], + "default": "person", + "help": "identity kind (default: person)", + "group": 1 + } + ] + } + }, + { + "name": "issue-location-token", + "parser": { + "prog": "sqb admin issue-location-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--expires-at" + ], + "dest": "admin_expires_at", + "required": true, + "help": "future expiry as an RFC3339 timestamp with timezone", + "group": 1 + } + ] + } + }, + { + "name": "issue-principal-token", + "parser": { + "prog": "sqb admin issue-principal-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "PRINCIPAL_ID", + "required": true, + "help": "principal UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--expires-at" + ], + "dest": "admin_expires_at", + "required": true, + "help": "future expiry as an RFC3339 timestamp with timezone", + "group": 1 + } + ] + } + }, + { + "name": "revoke-token", + "parser": { + "prog": "sqb admin revoke-token", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "TOKEN_ID", + "required": true, + "help": "credential UUID, never its bearer secret", + "group": 0 + } + ] + } + }, + { + "name": "rerun-run", + "parser": { + "prog": "sqb admin rerun-run", + "description": "Retry a terminal run using its pinned snapshot and settings, with an atomic security audit.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "SOURCE_RUN_ID", + "required": true, + "help": "terminal source run UUID", + "group": 0 + }, + { + "kind": "store", + "options": ["--run-id"], + "dest": "admin_event_id", + "required": true, + "help": "new run UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store_true", + "options": ["--from-failure"], + "dest": "admin_from_failure", + "nargs": 0, + "const": true, + "default": false, + "help": "retry exactly the failed and skipped nodes, excluding successful nodes", + "group": 1 + }, + { + "kind": "store", + "options": ["--priority"], + "dest": "admin_priority", + "type": "int", + "help": "signed queue priority; higher values run first (default: 0)", + "group": 1 + }, + { + "kind": "store", + "options": ["--reason"], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, + { + "name": "request-run", + "parser": { + "prog": "sqb admin request-run", + "description": "Queue a published job using privileged database access. Selection, image, settings and write claims come from the snapshot; the request and audit commit together.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": [], + "dest": "admin_job", + "metavar": "JOB", + "required": true, + "help": "published job name", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "admin_target", + "required": true, + "help": "published target name", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--run-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "run UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": ["--start-cursor-ts"], + "dest": "admin_start_cursor_ts", + "help": "backfill timestamp start (RFC3339); requires --end-cursor-ts", + "group": 1 + }, + { + "kind": "store", + "options": ["--end-cursor-ts"], + "dest": "admin_end_cursor_ts", + "help": "inclusive backfill timestamp end (RFC3339); requires --start-cursor-ts", + "group": 1 + }, + { + "kind": "store", + "options": ["--start-cursor-int"], + "dest": "admin_start_cursor_int", + "help": "backfill integer start; requires --end-cursor-int", + "group": 1 + }, + { + "kind": "store", + "options": ["--end-cursor-int"], + "dest": "admin_end_cursor_int", + "help": "inclusive backfill integer end; requires --start-cursor-int", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--priority" + ], + "dest": "admin_priority", + "type": "int", + "help": "signed queue priority; higher values run first (default: 0)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, + { + "name": "check-location", + "parser": { + "prog": "sqb admin check-location", + "description": "Request an outbound connection check on a published location using privileged database access.", + "groups": [ + {"title": "positional arguments"}, + {"title": "options"} + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "LOCATION_ID", + "required": true, + "help": "location UUID", + "group": 0 + }, + { + "kind": "store", + "options": ["--target"], + "dest": "admin_target", + "required": true, + "help": "published target name", + "group": 1 + }, + { + "kind": "store", + "options": ["--request-id"], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": ["--reason"], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the security audit", + "group": 1 + } + ] + } + }, + { + "name": "location-work", + "parser": { + "prog": "sqb admin location-work", + "description": "Read an outbound location request and its immutable result using privileged database access.", + "groups": [ + {"title": "positional arguments"}, + {"title": "options"} + ], + "actions": [ + { + "kind": "help", + "options": ["-h", "--help"], + "dest": "help", + "nargs": 0, + "default": {"suppress": true}, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "REQUEST_ID", + "required": true, + "help": "location work request UUID", + "group": 0 + } + ] + } + }, + { + "name": "cancel-run", + "parser": { + "prog": "sqb admin cancel-run", + "description": "Request cancellation through privileged database access; the fact, projection and security audit commit together. Active warehouse claims remain held until execution ends.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "RUN_ID", + "required": true, + "help": "run UUID", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--event-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the cancellation fact and security audit", + "group": 1 + } + ] + } + }, + { + "name": "change-trigger", + "parser": { + "prog": "sqb admin change-trigger", + "description": "Change runtime trigger state using privileged database access; the fact, projection and security audit commit together.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "admin_subject", + "metavar": "TRIGGER_ID", + "required": true, + "help": "current trigger UUID", + "group": 0 + }, + { + "kind": "store", + "options": [], + "dest": "admin_trigger_change", + "metavar": "CHANGE", + "choices": [ + "enable", + "pause", + "reset-default", + "set-cursor", + "reset-cursor" + ], + "required": true, + "help": "runtime change; reset-default restores the published status default", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--event-id" + ], + "dest": "admin_event_id", + "required": true, + "help": "request UUID; reuse it with identical arguments after an uncertain result", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--reason" + ], + "dest": "admin_reason", + "required": true, + "help": "nonblank reason recorded with the operator fact and security audit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--cursor" + ], + "dest": "admin_cursor", + "help": "RFC3339 timestamp with offset; required only for set-cursor", + "group": 1 + } + ] + } + } + ], + "choices_help": [ + { + "dest": "create-location", + "metavar": "create-location", + "help": "register a location with an atomic security audit record" + }, + { + "dest": "create-principal", + "metavar": "create-principal", + "help": "create a person or service account without granting permissions" + }, + { + "dest": "issue-location-token", + "metavar": "issue-location-token", + "help": "issue a bounded location credential, returned once as JSON" + }, + { + "dest": "issue-principal-token", + "metavar": "issue-principal-token", + "help": "issue a bounded principal credential, returned once as JSON" + }, + { + "dest": "revoke-token", + "metavar": "revoke-token", + "help": "revoke a credential by UUID" + }, + { + "dest": "request-run", + "metavar": "request-run", + "help": "queue a pinned published job with an atomic security audit" + }, + { + "dest": "rerun-run", + "metavar": "rerun-run", + "help": "retry a terminal run or exactly its failed and skipped nodes" + }, + { + "dest": "cancel-run", + "metavar": "cancel-run", + "help": "request run cancellation with an atomic audit and retry identity" + }, + { + "dest": "check-location", + "metavar": "check-location", + "help": "request an outbound location connection check with an atomic audit" + }, + { + "dest": "location-work", + "metavar": "location-work", + "help": "read a location work request and its result" + }, + { + "dest": "change-trigger", + "metavar": "change-trigger", + "help": "change trigger status or cursor with an atomic audit and retry identity" + } + ] + } + ] + } + }, + { + "name": "server", + "parser": { + "prog": "sqb server", + "description": "Run the native control API and scheduler without importing project Python.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "store", + "options": ["--launchers"], + "dest": "control_launchers", + "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--host" + ], + "dest": "control_host", + "default": "127.0.0.1", + "help": "IP address to listen on (default: 127.0.0.1)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--port" + ], + "dest": "control_port", + "default": 8000, + "type": "int", + "help": "TCP port, or 0 to choose a free port (default: 8000)", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-scheduler" + ], + "dest": "control_no_scheduler", + "nargs": 0, + "const": true, + "default": false, + "help": "serve the API without the embedded scheduler", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--openapi" + ], + "dest": "control_openapi", + "nargs": 0, + "const": true, + "default": false, + "help": "print the OpenAPI contract as JSON without opening a database", + "group": 1 + } + ] + } + }, + { + "name": "scheduler", + "parser": { + "prog": "sqb scheduler", + "description": "Run the lease-guarded native scheduler against the server's control database.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database-env" + ], + "dest": "control_database_env", + "default": "SQLBUILD_CONTROL_DATABASE_URL", + "help": "environment variable holding the control database URL (default: SQLBUILD_CONTROL_DATABASE_URL; SQLite when unset)", + "group": 1 + }, + { + "kind": "store", + "options": ["--launchers"], + "dest": "control_launchers", + "help": "installation-owned JSON file of local location/target launchers and concurrency limits", + "group": 1 + } + ] + } + }, { "name": "compile", "parser": { @@ -7592,6 +8582,26 @@ } ], "choices_help": [ + { + "dest": "location", + "metavar": "location", + "help": "compile and publish code-location snapshots over HTTP" + }, + { + "dest": "admin", + "metavar": "admin", + "help": "administer locations and credentials through privileged database access" + }, + { + "dest": "server", + "metavar": "server", + "help": "run the native control API and scheduler" + }, + { + "dest": "scheduler", + "metavar": "scheduler", + "help": "run a standalone lease-guarded scheduler" + }, { "dest": "cost", "metavar": "cost", @@ -7620,4 +8630,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index ef1b833fbb..3c079c4086 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -36,6 +36,8 @@ pub(crate) const MAXIMUM_BLANK_RUN: usize = 2; /// `CliNamespace` attributes the post-parse rules read and write. pub(crate) const COMMAND_DEST: &str = "command"; +pub(crate) const BUILD_COMMAND: &str = "build"; +pub(crate) const CONCURRENCY_DEST: &str = "concurrency"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_DEBUG_COMMAND: &str = "debug"; pub(crate) const DBT_PASSTHROUGH_COMMANDS: [&str; 4] = ["plan", "run", "build", "debug"]; diff --git a/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs new file mode 100644 index 0000000000..46d7b8ba46 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/job_settings_arguments.rs @@ -0,0 +1,32 @@ +//! Validate frozen build settings against the same table as the command-line parser. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::argv::_helpers::job_settings::{setting_argument, validate_mutex}; +use crate::argv::_helpers::normalization::cursor_error; +use crate::argv::_helpers::table::root_parser; +use crate::argv::constants::BUILD_COMMAND; + +/// Produce argument tokens, never shell text; execution identity is owned by the run request. +pub fn job_settings_arguments(settings: &BTreeMap) -> Result, String> { + let parser = root_parser() + .actions + .iter() + .flat_map(|action| &action.parsers) + .find(|(name, _)| name == BUILD_COMMAND) + .map(|(_, parser)| parser) + .ok_or_else(|| "build command is unavailable".to_owned())?; + let mut arguments: Vec = Vec::new(); + for (name, value) in settings { + if let Some(argument) = setting_argument(parser, name, value)? { + arguments.push(argument); + } + } + validate_mutex(parser, &arguments)?; + if let Some(message) = cursor_error(|name| settings.contains_key(name)) { + return Err(format!("job settings: {message}")); + } + Ok(arguments) +} diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 0dabda8f2b..d456903240 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,3 +1,4 @@ pub mod cli_namespace_defaults; +pub mod job_settings_arguments; pub mod parse_cli_arguments; pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs index d47a6702c5..62ba60f4db 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_parse_cli_arguments.rs @@ -1,6 +1,14 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::argv::main::job_settings_arguments::job_settings_arguments; use crate::argv::models::{ArgvOutcome, Value}; use crate::argv::tests::helpers::{parse, strings}; -use crate::argv::tests::test_types::{ExitTestCase, HelpTextTestCase, ParsedValueTestCase}; +use crate::argv::tests::test_types::{ + ExitTestCase, HelpTextTestCase, JobSettingsRejectionTestCase, JobSettingsTestCase, + ParsedValueTestCase, +}; #[test] fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values() { @@ -41,6 +49,35 @@ fn given_accepted_arguments_when_parsing_then_attributes_hold_the_parsed_values( attribute: "contract_command", expected_value: Value::Str("diff".to_owned()), }, + ParsedValueTestCase { + description: "an explicit failed-node rerun never becomes a full rerun", + argv: &[ + "admin", + "rerun-run", + "00000000-0000-0000-0000-000000000001", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Retry failed orders", + "--from-failure", + ], + attribute: "admin_from_failure", + expected_value: Value::Bool(true), + }, + ParsedValueTestCase { + description: "a full rerun is the default without failure narrowing", + argv: &[ + "admin", + "rerun-run", + "00000000-0000-0000-0000-000000000001", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Retry all orders", + ], + attribute: "admin_from_failure", + expected_value: Value::Bool(false), + }, ]; for test_case in test_cases { @@ -172,3 +209,91 @@ fn given_command_help_when_parsing_then_hidden_options_are_left_out() { assert_eq!(absent, test_case.expected_absent, "{help}"); } } + +#[test] +fn given_job_settings_when_validating_then_values_survive_the_build_parser() { + let test_cases = [ + JobSettingsTestCase { + description: "inverse flags and option-like values survive the build parser", + settings: BTreeMap::from([ + ("concurrency".into(), json!(3)), + ("load_sources".into(), json!(false)), + ("reload".into(), json!(false)), + ("run_tests".into(), json!(false)), + ("full_refresh".into(), json!(true)), + ("warehouse".into(), json!("--full-refresh")), + ]), + expected_values: vec![ + ("concurrency", Value::Int("3".into())), + ("load_sources", Value::Bool(false)), + ("reload", Value::Bool(false)), + ("run_tests", Value::Bool(false)), + ("full_refresh", Value::Bool(true)), + ("warehouse", Value::Str("--full-refresh".into())), + ], + expected_empty: false, + }, + JobSettingsTestCase { + description: "default-valued flags do not become explicit options", + settings: BTreeMap::from([ + ("run_tests".into(), json!(true)), + ("full_refresh".into(), json!(false)), + ("debug".into(), json!(false)), + ]), + expected_values: vec![], + expected_empty: true, + }, + ]; + for test_case in test_cases { + let settings = job_settings_arguments(&test_case.settings).expect(test_case.description); + assert_eq!( + settings.is_empty(), + test_case.expected_empty, + "{}", + test_case.description + ); + let mut arguments: Vec = vec!["build".to_owned()]; + arguments.extend(settings); + let arguments: Vec<&str> = arguments.iter().map(String::as_str).collect(); + let parsed = parse(&arguments).parsed.expect(test_case.description); + for (name, expected_value) in test_case.expected_values { + assert_eq!( + parsed.get(name), + Some(&expected_value), + "{}", + test_case.description + ); + } + } +} + +#[test] +fn given_conflicting_job_settings_when_validating_then_options_are_rejected() { + let test_cases = [ + JobSettingsRejectionTestCase { + description: "no-load conflicts with reload", + settings: BTreeMap::from([ + ("load_sources".into(), json!(false)), + ("reload".into(), json!(true)), + ]), + expected_error: "mutually exclusive", + }, + JobSettingsRejectionTestCase { + description: "load conflicts with reload", + settings: BTreeMap::from([ + ("load_sources".into(), json!(true)), + ("reload".into(), json!(true)), + ]), + expected_error: "mutually exclusive", + }, + ]; + for test_case in test_cases { + assert!( + job_settings_arguments(&test_case.settings) + .expect_err(test_case.description) + .contains(test_case.expected_error), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index 4452d1689e..793e4a87a3 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -1,3 +1,6 @@ +use std::collections::BTreeMap; + +use serde_json::Value as Json; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use sqlbuild_core::dbt_cli::models::DbtParsedFlags; @@ -63,3 +66,18 @@ pub(super) struct DbtExecutionTestCase { pub(super) args: &'static [&'static str], pub(super) expected_parsed: Result, } + +/// Authored settings and the build namespace or validation failure they produce. +pub(super) struct JobSettingsTestCase { + pub(super) description: &'static str, + pub(super) settings: BTreeMap, + pub(super) expected_values: Vec<(&'static str, Value)>, + pub(super) expected_empty: bool, +} + +/// Mutually exclusive authored options rejected before a snapshot is published. +pub(super) struct JobSettingsRejectionTestCase { + pub(super) description: &'static str, + pub(super) settings: BTreeMap, + pub(super) expected_error: &'static str, +} diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 550b31ebe3..247242fd45 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -515,6 +515,27 @@ pub(crate) const DEBUG_ROUTE: Route = routed( }, ); +const LOCATION_ARGUMENTS: [Field; 9] = [ + field("project_dir", Source::EffectiveProjectDir), + SELECTED_TARGET, + same("location_id"), + attr("server_url", "location_server_url"), + attr("generation", "location_generation"), + attr("image_id", "location_image_id"), + attr("action", "location_command"), + attr("run_id", "location_run_id"), + attr("claim_id", "location_claim_id"), +]; +pub(crate) const LOCATION_ROUTE: Route = routed( + "run_location", + "run_location_command", + &LOCATION_ARGUMENTS, + CallShape { + slot_positional: 0, + implementation_keywords: true, + }, +); + const JANITOR_FIELDS: [Field; 11] = [ PROJECT_DIR, NO_COLOR, diff --git a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs index 29de9f0374..bb91c6290a 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs @@ -6,9 +6,9 @@ use crate::dispatch::_helpers::subcommands::{ use crate::dispatch::constants::{ AUDIT_ROUTE, BUILD_ROUTE, CHECK_ROUTE, CLONE_ROUTE, COMPILE_ROUTE, CONTRACT_ROUTE, COST_ROUTE, DAG_ROUTE, DEBUG_ROUTE, DIFF_ROUTE, FORMAT_ROUTE, FRESHNESS_ROUTE, INIT_ROUTE, JANITOR_ROUTE, - LINEAGE_ROUTE, LOAD_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, RECALL_COMMAND_NAME, - RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, - TEST_ROUTE, + LINEAGE_ROUTE, LOAD_ROUTE, LOCATION_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, + RECALL_COMMAND_NAME, RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, + SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, }; use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; @@ -55,6 +55,7 @@ pub fn command_dispatch(facts: &CommandFacts) -> Dispatch { RECALL_COMMAND_NAME => run(&RECALL_ROUTE), "cost" => run(&COST_ROUTE), "debug" => run(&DEBUG_ROUTE), + "location" => run(&LOCATION_ROUTE), "janitor" => run(&JANITOR_ROUTE), "init" => run(&INIT_ROUTE), "playground" => run(&PLAYGROUND_ROUTE), diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index e6f689c242..ff3bfcc8ea 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -10,7 +10,11 @@ use crate::launcher::_helpers::messages::{print_version, report_missing_python_c use crate::launcher::constants::{ HANDOFF_FAILURE_EXIT_CODE, LAUNCH_RECORD_OPTION, VERSION_OPTIONS, }; +use crate::native_commands::main::parse_native_invocation::parse_native_invocation; +use crate::native_commands::main::run_control_invocation::run_control_invocation; +use crate::native_commands::main::run_location_invocation::run_location_invocation; use crate::native_commands::main::run_native_invocation::run_native_invocation; +use crate::native_commands::models::NativeParse; use crate::replay::main::serve_reused_compile::serve_reused_compile; /// The launch record path and key for these arguments, when the directory is known. @@ -38,11 +42,25 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - print_version(); return 0; } + let native = parse_native_invocation(&arguments); + if let Ok(NativeParse::Exit(code)) = &native { + return *code; + } + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_control_invocation(namespace) + { + return code; + } let program: PathBuf = std::env::current_exe().unwrap_or_default(); let Some(script) = python_cli(&program) else { report_missing_python_cli(&program); return HANDOFF_FAILURE_EXIT_CODE; }; + if let Ok(NativeParse::Invocation { namespace, .. }) = &native + && let Some(code) = run_location_invocation(namespace, &script) + { + return code; + } let mut handed: Vec = Vec::with_capacity(arguments.len() + 1); let record: Option<(PathBuf, String)> = launch_record(&program, &script, &arguments); if let Some((path, key)) = &record @@ -50,7 +68,12 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - { return exit_code(code); } - if let Ok(code) = run_native_invocation(&arguments, &script) { + if let Ok(NativeParse::Invocation { + namespace, + use_color, + }) = native + && let Ok(code) = run_native_invocation(&namespace, &script, use_color) + { return code; } if let Some((path, _)) = record { diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 538dec3263..5364b9c483 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -10,3 +10,4 @@ pub mod dispatch; pub mod launcher; pub mod native_commands; pub mod replay; +pub(crate) mod workers; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 1ab59675b6..f78d6f1874 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -1,7 +1,17 @@ -//! Which parsed invocations run without Python, decided from the parsed values alone. +//! Resolve native invocations and control-service settings from parsed arguments. +use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; +use chrono::{DateTime, Utc}; +use serde_json::json; +use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest, RunWindow}; +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + use crate::argv::constants::{JSON_OPTION, REVIEW_PROG}; use crate::argv::models::{ArgvOutcome, Namespace, Value}; use crate::dispatch::main::invocation_policy::invocation_policy; @@ -12,7 +22,8 @@ use crate::native_commands::constants::{ }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{ - CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, + ControlAdmin, CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, + SkillsOptions, }; use sqlbuild_core::cost_command::constants::DESCENDING_ORDER; @@ -25,17 +36,238 @@ pub(crate) fn review_json_failure(arguments: &[String], outcome: &ArgvOutcome) - .is_some_and(|failure| failure.prog == REVIEW_PROG) } -fn text(namespace: &Namespace, name: &str) -> Option { +pub(crate) fn text(namespace: &Namespace, name: &str) -> Option { match namespace.get(name) { Some(Value::Str(text) | Value::Path(text)) => Some(text.clone()), _ => None, } } -fn flag(namespace: &Namespace, name: &str) -> bool { +pub(crate) fn flag(namespace: &Namespace, name: &str) -> bool { namespace.get(name) == Some(&Value::Bool(true)) } +pub(crate) fn control_admin(namespace: &Namespace) -> Result { + let value = text(namespace, "admin_subject").unwrap_or_default(); + let action = text(namespace, "admin_command"); + if action.as_deref() == Some("create-location") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("location name is required".into())); + } + return Ok(ControlAdmin::CreateLocation(value)); + } + if action.as_deref() == Some("create-principal") { + if value.trim().is_empty() { + return Err(ControlError::Invalid("principal name is required".into())); + } + let kind = match text(namespace, "admin_principal_kind").as_deref() { + Some("service_account") => PrincipalKind::ServiceAccount, + _ => PrincipalKind::Person, + }; + return Ok(ControlAdmin::CreatePrincipal(value, kind)); + } + let id = Uuid::parse_str(&value) + .map_err(|_| ControlError::Invalid("a valid subject UUID is required".into()))?; + match action.as_deref() { + Some("location-work") => Ok(ControlAdmin::LocationWork(id)), + Some("check-location") => { + let request_id = + Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) + .map_err(|_| ControlError::Invalid("--request-id must be a UUID".into()))?; + let target = text(namespace, "admin_target").unwrap_or_default(); + let reason = text(namespace, "admin_reason").unwrap_or_default(); + if target.trim().is_empty() || reason.trim().is_empty() { + return Err(ControlError::Invalid( + "target and reason must not be blank".into(), + )); + } + Ok(ControlAdmin::RequestLocationWork(LocationWorkRequest { + request_id, + location_id: id, + target, + reason, + work: LocationWork::CheckConnection {}, + })) + } + Some("change-trigger" | "cancel-run" | "request-run" | "rerun-run") => { + let identity_option = if matches!(action.as_deref(), Some("request-run" | "rerun-run")) + { + "--run-id" + } else { + "--event-id" + }; + let event_id = Uuid::parse_str(&text(namespace, "admin_event_id").unwrap_or_default()) + .map_err(|_| ControlError::Invalid(format!("{identity_option} must be a UUID")))?; + let reason = text(namespace, "admin_reason").unwrap_or_default(); + if reason.trim().is_empty() { + return Err(ControlError::Invalid("--reason must not be blank".into())); + } + if matches!(action.as_deref(), Some("request-run" | "rerun-run")) { + let priority = match namespace.get("admin_priority") { + Some(Value::Int(value)) => value.parse::().map_err(|_| { + ControlError::Invalid("--priority must fit a signed 64-bit integer".into()) + })?, + _ => 0, + }; + if action.as_deref() == Some("rerun-run") { + return Ok(ControlAdmin::Rerun(RerunRequest { + run_id: event_id, + source_run_id: id, + from_failure: flag(namespace, "admin_from_failure"), + reason, + priority, + })); + } + let target = text(namespace, "admin_target").unwrap_or_default(); + let job_key = text(namespace, "admin_job").unwrap_or_default(); + if target.trim().is_empty() || job_key.trim().is_empty() { + return Err(ControlError::Invalid( + "target and job must not be blank".into(), + )); + } + let window = match ( + text(namespace, "admin_start_cursor_ts"), + text(namespace, "admin_end_cursor_ts"), + text(namespace, "admin_start_cursor_int"), + text(namespace, "admin_end_cursor_int"), + ) { + (None, None, None, None) => None, + (Some(start), Some(end), None, None) => Some(RunWindow::Timestamp { + start: DateTime::parse_from_rfc3339(&start) + .map_err(|_| { + ControlError::Invalid( + "backfill start must be an RFC3339 timestamp with timezone" + .into(), + ) + })? + .with_timezone(&Utc), + end: DateTime::parse_from_rfc3339(&end) + .map_err(|_| { + ControlError::Invalid( + "backfill end must be an RFC3339 timestamp with timezone" + .into(), + ) + })? + .with_timezone(&Utc), + }), + (None, None, Some(start), Some(end)) => Some(RunWindow::Integer { + start: start.parse().map_err(|_| { + ControlError::Invalid( + "backfill start must be a signed 64-bit integer".into(), + ) + })?, + end: end.parse().map_err(|_| { + ControlError::Invalid( + "backfill end must be a signed 64-bit integer".into(), + ) + })?, + }), + _ => { + return Err(ControlError::Invalid( + "backfill requires both start and end of exactly one cursor type" + .into(), + )); + } + }; + return Ok(ControlAdmin::RequestJobRun(JobRunRequest { + run_id: event_id, + location_id: id, + target, + job_key, + reason, + priority, + window, + })); + } + if action.as_deref() == Some("cancel-run") { + return Ok(ControlAdmin::CancelRun { + event_id, + run_id: id, + reason, + }); + } + let mut change = json!({ + "kind": text(namespace, "admin_trigger_change").unwrap_or_default().replace('-', "_"), + }); + let cursor = text(namespace, "admin_cursor"); + if let Some(cursor) = &cursor { + change["time"] = json!(cursor); + } + let change: TriggerChange = serde_json::from_value(change) + .map_err(|error| ControlError::Invalid(format!("trigger change: {error}")))?; + if cursor.is_some() && !matches!(change, TriggerChange::SetCursor { .. }) { + return Err(ControlError::Invalid( + "--cursor is only valid for set-cursor".into(), + )); + } + Ok(ControlAdmin::ChangeTrigger { + event_id, + trigger_id: id, + change, + reason, + }) + } + Some("issue-location-token" | "issue-principal-token") => { + let expiry = DateTime::parse_from_rfc3339( + &text(namespace, "admin_expires_at").unwrap_or_default(), + ) + .map_err(|_| ControlError::Invalid("--expires-at must be an RFC3339 timestamp".into()))? + .with_timezone(&Utc); + if expiry <= Utc::now() { + return Err(ControlError::Invalid( + "token expiry must be in the future".into(), + )); + } + let scope = match action.as_deref() { + Some("issue-principal-token") => CredentialScope::Principal { principal_id: id }, + _ => CredentialScope::Location { location_id: id }, + }; + Ok(ControlAdmin::IssueToken(scope, expiry)) + } + Some("revoke-token") => Ok(ControlAdmin::RevokeToken(id)), + _ => Err(ControlError::Invalid( + "an administration command is required".into(), + )), + } +} + +pub(crate) fn control_address( + namespace: &Namespace, + standalone: bool, +) -> Result, ControlError> { + if standalone { + return Ok(None); + } + let host: IpAddr = text(namespace, "control_host") + .unwrap_or_else(|| "127.0.0.1".into()) + .parse() + .map_err(|_| ControlError::Invalid("--host must be an IP address".into()))?; + let port = match namespace.get("control_port") { + Some(Value::Int(value)) => value + .parse::() + .map_err(|_| ControlError::Invalid("--port must be between 0 and 65535".into()))?, + _ => 8000, + }; + Ok(Some(SocketAddr::new(host, port))) +} + +pub(crate) fn control_database(namespace: &Namespace) -> Result { + let variable = text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()); + match std::env::var(&variable) { + Ok(value) => Ok(value), + Err(std::env::VarError::NotPresent) => Ok(PathBuf::from( + text(namespace, "project_dir").unwrap_or_else(|| ".".into()), + ) + .join(".sqlbuild/control.sqlite3") + .to_string_lossy() + .into_owned()), + Err(std::env::VarError::NotUnicode(_)) => Err(ControlError::Invalid( + "control database variable must be UTF-8".into(), + )), + } +} + /// The strings of a list attribute, or `None` when it holds anything else. fn texts(namespace: &Namespace, name: &str) -> Option> { match namespace.get(name) { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 90297029d9..27c20ee723 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -1,29 +1,413 @@ //! Running a Python-free command: its checks, then its output with the observed services. +use std::future::Future; use std::io; use std::path::{Path, PathBuf}; -use std::time::SystemTime; +use std::process::Stdio; +use std::time::{Duration, SystemTime}; -use crate::argv::models::Stream; +use chrono::{DateTime, Utc}; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; +use sqlbuild_orchestration::platform::main::connect_platform_webhook::connect_platform_webhook; +use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; +use sqlbuild_orchestration::platform::main::run_platform_sink::run_platform_sink; +use sqlbuild_orchestration::platform::main::run_watchdog::run_watchdog; +use sqlbuild_orchestration::platform::models::{LocationHealth, LocationHealthReport}; +use sqlbuild_orchestration::snapshots::main::next_remote_generation::next_remote_generation; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::migrate_store::migrate_store; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; +use tokio::process::{Child, Command}; +use tokio::sync::{mpsc, oneshot, watch}; +use tokio::task::JoinSet; +use tokio::time::MissedTickBehavior; +use uuid::Uuid; + +use crate::argv::models::{Namespace, Stream, Value}; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; use crate::compute_logs::main::abandon_capture::abandon_capture; use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; +use crate::native_commands::_helpers::decision::{control_database, flag, text}; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; -use crate::native_commands::_helpers::paths::project_location; +use crate::native_commands::_helpers::paths::{location_database_files, project_location}; use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; use crate::native_commands::_helpers::rules::run_rules; use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; -use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; -use crate::native_commands::constants::{UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND}; +use crate::native_commands::_helpers::terminal::{ + finish_lost_stdout, terminal_columns, write_output, +}; +use crate::native_commands::constants::{ + COLUMNS_VARIABLE, FALLBACK_COLUMNS, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, +}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationServe, + LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; +use crate::native_commands::types::LocationPublication; +use crate::workers::main::open_workers::open_workers; +use crate::workers::models::Workers; + +/// Request one publication without transferring interpreter ownership to a cancellable future. +pub(crate) async fn compile_generation( + requests: &mpsc::Sender, + generation: i64, + sources: &[sqlbuild_cache::project_snapshot::models::PathStamp], + stopped: impl Future>, +) -> Result>)>, ControlError> { + let (reply, publication) = oneshot::channel(); + requests + .send(LocationCompileRequest { + generation, + python_sources: sources + .iter() + .filter(|stamp| stamp.relative_path.ends_with(".py")) + .cloned() + .collect(), + reply, + }) + .await + .map_err(|_| ControlError::Invalid("location compiler stopped".into()))?; + tokio::pin!(stopped); + let result = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = publication => result.map_err(|_| ControlError::Invalid("location compiler stopped before replying".into()))?, + }; + let record = match result { + Ok(record) => record, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!( + "Location compiler failed; generation {generation} may already be published: {error}\n" + ), + )])); + None + } + }; + let published = record.is_some(); + let refresh_at = record.as_ref().and_then(|(_, refresh_at)| *refresh_at); + if let Some((record, _)) = record { + let output = serde_json::to_string(&record) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if write_output(&[(Stream::Stdout, &format!("{output}\n"))]).is_some() { + return Err(ControlError::Invalid(format!( + "Snapshot {} was published, but result delivery failed", + record.snapshot_id + ))); + } + } + Ok(Some((published, refresh_at))) +} + +/// The child is retained by its owning task while this exchange can be cancelled. +pub(crate) async fn exchange_location_generation( + compiler: &mut LocationCompiler, + generation: i64, +) -> Result, ControlError> { + compiler + .stdin + .write_all(format!("{generation}\n").as_bytes()) + .await?; + compiler.stdin.flush().await?; + let mut line = String::new(); + if compiler.stdout.read_line(&mut line).await? == 0 { + return Err(ControlError::Invalid( + "location interpreter exited before replying".into(), + )); + } + serde_json::from_str(&line).map_err(|_| { + ControlError::Invalid("location interpreter returned an invalid publication record".into()) + }) +} + +/// Health IO remains native even while a user-code child holds its interpreter's GIL. +pub(crate) async fn report_location_health( + options: &LocationServe, + bearer: &str, + session_id: Uuid, + health: LocationHealth, +) { + let report = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: options.target.clone(), + health, + }; + if let Err(error) = + report_remote_location_health((&options.server_url, bearer), options.location_id, &report) + .await + { + drop(write_output(&[( + Stream::Stderr, + &format!("Location health report failed: {error}\n"), + )])); + } +} + +pub(crate) async fn location_heartbeats( + options: &LocationServe, + bearer: &str, + session_id: Uuid, +) -> Result<(), ControlError> { + let mut clock = tokio::time::interval(Duration::from_secs(30)); + clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + clock.tick().await; + report_location_health(options, bearer, session_id, LocationHealth::Heartbeat {}).await; + } +} + +pub(crate) fn location_options(namespace: &Namespace) -> Result { + if namespace + .get("location_generation") + .is_some_and(|value| *value != Value::None) + { + return Err(ControlError::Invalid( + "serve derives --generation from the server; do not supply it".into(), + )); + } + let project_dir = text(namespace, "project_dir") + .map_or_else(std::env::current_dir, |directory| { + std::fs::canonicalize(directory) + })?; + if !project_dir.is_dir() { + return Err(ControlError::Invalid("project directory must exist".into())); + } + let location_id = text(namespace, "location_id") + .unwrap_or_default() + .parse() + .map_err(|_| ControlError::Invalid("location ID must be a UUID".into()))?; + let target = text(namespace, "target").unwrap_or_default(); + if target.trim().is_empty() { + return Err(ControlError::Invalid("target must not be blank".into())); + } + Ok(LocationServe { + project_dir, + server_url: text(namespace, "location_server_url").unwrap_or_default(), + location_id, + target, + image_id: text(namespace, "location_image_id"), + }) +} + +/// Reconcile from the server after restarts or lost acknowledgements; transient outages retry. +pub(crate) async fn location_generation( + options: &LocationServe, + bearer: &str, +) -> Result, ControlError> { + match next_remote_generation( + &options.server_url, + bearer, + options.location_id, + &options.target, + ) + .await + { + Ok(generation) => Ok(Some(generation)), + Err(error) => { + if matches!(&error, ClientError::Configuration(_)) + || matches!(&error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + drop(write_output(&[( + Stream::Stderr, + &format!( + "Location generation lookup failed; retaining last good snapshot and retrying: {error}\n" + ), + )])); + Ok(None) + } + } +} + +/// The installed interpreter owns user imports; the supervisor never imports Python. +pub(crate) fn location_runtime_child(options: &LocationServe, script: &Path) -> io::Result { + let mut command = location_child(options, script); + command + .args([ + "location", + "runtime", + "--server", + &options.server_url, + "--location-id", + ]) + .arg(options.location_id.to_string()) + .args(["--target", &options.target]); + if let Some(image) = &options.image_id { + command.args(["--image-id", image]); + } + command.stdin(Stdio::piped()).stdout(Stdio::piped()).spawn() +} + +/// Capture the ordinary debug check's exact local file paths without contaminating publication stdout. +pub(crate) async fn check_location_connection( + options: &LocationServe, + script: &Path, + stopped: impl Future>, +) -> Result)>, ControlError> { + let mut child = location_child(options, script) + .args(["debug", "--target", &options.target, "--json"]) + .stdout(Stdio::piped()) + .spawn()?; + let mut stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("location debug output is unavailable".into()))?; + let mut output: Vec = Vec::new(); + tokio::pin!(stopped); + let connected = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = async { tokio::try_join!(child.wait(), stdout.read_to_end(&mut output)) } => result?.0.success(), + }; + let files: Vec = match serde_json::from_slice::(&output) + .and_then(|report| serde_json::from_value(report["local_files"].clone())) + { + Ok(files) => files, + Err(_) => { + drop(write_output(&[( + Stream::Stderr, + "Location connection check did not produce a valid report; retaining the published snapshot.\n", + )])); + return Ok(Some((false, Vec::new()))); + } + }; + Ok(Some(( + connected, + location_database_files(&options.project_dir, files), + ))) +} + +fn location_child(options: &LocationServe, script: &Path) -> Command { + let mut command = Command::new(script); + command.arg("--project-dir").arg(&options.project_dir); + if terminal_columns().is_err() { + command.env(COLUMNS_VARIABLE, FALLBACK_COLUMNS.to_string()); + } + command + .stdin(Stdio::null()) + .stderr(Stdio::inherit()) + .kill_on_drop(true); + command +} + +/// Health production and delivery remain native and independent of scheduler leadership. +pub(crate) fn start_platform_roles( + tasks: &mut JoinSet>, + (store, shutdown): (&ControlStore, &watch::Receiver), +) -> Result<(), ControlError> { + let sink = match std::env::var(PLATFORM_WEBHOOK_VARIABLE) { + Ok(endpoint) => Some(connect_platform_webhook(&endpoint)?), + Err(std::env::VarError::NotPresent) => None, + Err(_) => { + return Err(ControlError::Invalid( + "platform webhook configuration must be UTF-8".into(), + )); + } + }; + drop(write_output(&[( + Stream::Stderr, + "Starting platform watchdog...\n", + )])); + let watchdog_store = store.clone(); + let watchdog_shutdown = shutdown.clone(); + tasks.spawn( + async move { run_watchdog(&watchdog_store, Uuid::now_v7(), watchdog_shutdown).await }, + ); + drop(write_output(&[( + Stream::Stderr, + "Platform watchdog started\n", + )])); + if let Some(sink) = sink { + drop(write_output(&[( + Stream::Stderr, + "Starting platform webhook...\n", + )])); + let sink_store = store.clone(); + let sink_shutdown = shutdown.clone(); + tasks.spawn(async move { run_platform_sink(&sink_store, sink, sink_shutdown).await }); + drop(write_output(&[( + Stream::Stderr, + "Platform webhook started\n", + )])); + } + Ok(()) +} + +/// Open and migrate the privileged store, reporting progress without contaminating JSON stdout. +pub(crate) async fn open_control_store( + namespace: &Namespace, +) -> Result { + let url = control_database(namespace)?; + drop(write_output(&[( + Stream::Stderr, + "Opening control store...\n", + )])); + let store = open_store(&url).await?; + migrate_store(&store).await?; + drop(write_output(&[(Stream::Stderr, "Control store ready\n")])); + Ok(store) +} + +/// Decode privileged CLI configuration before handing process ownership to the workers domain. +pub(crate) fn open_control_workers(namespace: &Namespace) -> Result { + open_workers( + text(namespace, "control_launchers").as_deref(), + text(namespace, "control_database_env") + .unwrap_or_else(|| "SQLBUILD_CONTROL_DATABASE_URL".into()), + !flag(namespace, "control_no_scheduler"), + ) +} + +pub(crate) async fn shutdown_signal() -> Result<(), ControlError> { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result?, + _ = terminate.recv() => {}, + } + } + #[cfg(not(unix))] + tokio::signal::ctrl_c().await?; + Ok(()) +} + +/// Stop sibling roles on a signal or role failure, then drain their shutdown results. +pub(crate) async fn supervise_control_roles( + mut tasks: JoinSet>, + stop: watch::Sender, +) -> Result<(), ControlError> { + let mut result = tokio::select! { + result = tasks.join_next() => match result { + Some(result) => result.map_err(ControlError::from).and_then(std::convert::identity), + None => Ok(()), + }, + result = shutdown_signal() => result, + }; + stop.send_replace(true); + while let Some(joined) = tasks.join_next().await { + let joined = joined + .map_err(ControlError::from) + .and_then(std::convert::identity); + result = result.and(joined); + } + result +} /// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. fn open_capture( diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index 302e8a5a66..78b96ec94b 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -1,11 +1,153 @@ //! `pathlib.Path` construction and `Path.resolve()` over native paths. use std::path::{Component, Path, PathBuf}; +use std::time::Duration; +use sqlbuild_cache::digest::constants::{ + EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, PRESENCE_ONLY_FILE_SUFFIXES, +}; +use sqlbuild_cache::project_snapshot::main::snapshot_project_files::snapshot_project_files; +use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; +use sqlbuild_core::git_revisions::main::head_commit::head_commit; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; +use tokio::time::Instant; use crate::native_commands::errors::PythonNeeded; -use crate::native_commands::models::ProjectLocation; +use crate::native_commands::models::{LocationWatch, ProjectLocation}; +use crate::replay::main::output_file_identities::output_file_identities; + +/// Observe the same project inputs as compiler reuse, excluding redirected command output. +pub(crate) fn location_watch(root: &Path) -> LocationWatch { + let rules = location_rules(); + LocationWatch { + observed: location_inputs(root, &rules, &[]), + rules, + local_files: Vec::new(), + attempted: None, + changed_at: tokio::time::Instant::now(), + retry_at: None, + } +} + +/// Coalesce source edits and retry deadlines without replacing the retained attempted input. +pub(crate) fn observe_location_inputs( + root: &Path, + mut watch: LocationWatch, +) -> (LocationWatch, bool) { + let current = location_inputs(root, &watch.rules, &watch.local_files); + if watch.observed != current { + watch.observed = current; + watch.changed_at = Instant::now(); + } + if watch.attempted.as_ref() == Some(&watch.observed) + && watch.retry_at.is_none_or(|retry| Instant::now() < retry) + { + return (watch, false); + } + if watch.attempted.is_some() && watch.changed_at.elapsed() < Duration::from_millis(250) { + return (watch, false); + } + watch.attempted = Some(watch.observed.clone()); + (watch, true) +} + +fn location_rules() -> SnapshotRules { + SnapshotRules { + excluded: EXCLUDED_DIRECTORIES + .iter() + .map(ToString::to_string) + .collect(), + excluded_root: EXCLUDED_ROOT_DIRECTORIES + .iter() + .map(ToString::to_string) + .collect(), + presence_suffixes: PRESENCE_ONLY_FILE_SUFFIXES + .iter() + .map(ToString::to_string) + .collect(), + output_files: output_file_identities(), + } +} + +/// Git commits matter to publication metadata even when no working-tree file changes. +pub(crate) fn location_inputs( + root: &Path, + rules: &SnapshotRules, + local_files: &[PathBuf], +) -> (Vec, Option) { + let mut stamps = snapshot_project_files(root, rules); + stamps.retain(|stamp| { + !rules + .presence_suffixes + .iter() + .any(|suffix| stamp.relative_path.ends_with(suffix)) + }); + stamps = exclude_location_files(stamps, root, local_files); + #[cfg(windows)] + { + let output: Vec = + [same_file::Handle::stdout(), same_file::Handle::stderr()] + .into_iter() + .flatten() + .filter(|handle| { + handle + .as_file() + .metadata() + .is_ok_and(|metadata| metadata.is_file()) + }) + .collect(); + stamps.retain(|stamp| { + same_file::Handle::from_path(root.join(&stamp.relative_path)) + .map_or(true, |handle| !output.contains(&handle)) + }); + } + stamps.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + (stamps, head_commit(root)) +} + +/// Apply resolved exclusions to retained observations, never a fresh snapshot of possible source edits. +pub(crate) fn exclude_location_databases( + root: &Path, + files: Vec, + mut watch: LocationWatch, +) -> LocationWatch { + watch.local_files.extend(files); + watch.local_files.sort(); + watch.local_files.dedup(); + watch.observed.0 = exclude_location_files(watch.observed.0, root, &watch.local_files); + watch.attempted = watch.attempted.map(|(stamps, commit)| { + ( + exclude_location_files(stamps, root, &watch.local_files), + commit, + ) + }); + watch +} + +fn exclude_location_files( + mut stamps: Vec, + root: &Path, + local_files: &[PathBuf], +) -> Vec { + stamps.retain(|stamp| !local_files.contains(&root.join(&stamp.relative_path))); + stamps +} + +pub(crate) fn location_database_files(root: &Path, files: Vec) -> Vec { + let mut paths: Vec = Vec::new(); + for file in files { + let authored = pathlib_path(&root.join(file)); + let resolved = project_real_path(&authored); + for path in [authored, resolved] { + for suffix in ["", ".wal", "-wal", "-shm", "-journal"] { + let mut file = path.as_os_str().to_os_string(); + file.push(suffix); + paths.push(PathBuf::from(file)); + } + } + } + paths +} /// `Path(text)`: repeated separators, `.` parts and trailing separators dropped. pub(crate) fn pathlib_path(text: &Path) -> PathBuf { diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 867ce8bb82..65784a5e9b 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -41,6 +41,9 @@ pub(crate) const RUNNING_BUILD_STATUS: &str = "running"; pub(crate) const CLEAN_COMMAND: &str = "clean"; pub(crate) const INIT_COMMAND: &str = "init"; pub(crate) const SKILLS_COMMAND: &str = "skills"; +pub(crate) const SERVER_COMMAND: &str = "server"; +pub(crate) const SCHEDULER_COMMAND: &str = "scheduler"; +pub(crate) const ADMIN_COMMAND: &str = "admin"; pub(crate) const RULES_COMMAND: &str = "rules"; pub(crate) const RULES_LIST_ACTION: &str = "list"; pub(crate) const RULES_SHOW_ACTION: &str = "show"; @@ -87,3 +90,6 @@ pub(crate) const EXIT_FLUSH_FAILURE: &str = "Exception ignored in: <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>"; pub(crate) const EXIT_FLUSH_FAILURE_STATUS: i32 = 120; pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; +/// Reserve half the location work claim for acknowledgement retries after killing a callback. +pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; +pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs new file mode 100644 index 0000000000..2a20ef09b2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs @@ -0,0 +1,83 @@ +//! One task owns a warm interpreter; cancelled requests and source restarts always reap it. + +use std::path::Path; + +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::BufReader; +use tokio::sync::mpsc; + +use crate::native_commands::_helpers::execution::{ + exchange_location_generation, location_runtime_child, +}; +use crate::native_commands::models::{LocationCompileRequest, LocationCompiler, LocationServe}; + +pub(crate) async fn run_location_compiler( + options: &LocationServe, + script: &Path, + mut requests: mpsc::Receiver, +) -> Result<(), ControlError> { + let mut compiler: Option = None; + let result = async { + while let Some(mut request) = requests.recv().await { + if compiler + .as_ref() + .is_some_and(|compiler| compiler.python_sources != request.python_sources) + { + stop_compiler(compiler.take()).await?; + } + if compiler.is_none() { + compiler = Some(open_compiler(options, script, request.python_sources)?); + } + let active = compiler + .as_mut() + .ok_or_else(|| ControlError::Invalid("location compiler is unavailable".into()))?; + let result = tokio::select! { + _ = request.reply.closed() => break, + result = exchange_location_generation(active, request.generation) => result, + }; + let failed = result.is_err(); + let _ = request + .reply + .send(result.map_err(|error| error.to_string())); + if failed { + stop_compiler(compiler.take()).await?; + } + } + Ok(()) + } + .await; + stop_compiler(compiler).await?; + result +} + +fn open_compiler( + options: &LocationServe, + script: &Path, + python_sources: Vec, +) -> Result { + let mut child = location_runtime_child(options, script)?; + let stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("location interpreter input is unavailable".into()))?; + let stdout = child.stdout.take().ok_or_else(|| { + ControlError::Invalid("location interpreter output is unavailable".into()) + })?; + Ok(LocationCompiler { + child, + stdin, + stdout: BufReader::new(stdout), + python_sources, + }) +} + +async fn stop_compiler(compiler: Option) -> Result<(), ControlError> { + if let Some(mut compiler) = compiler { + if compiler.child.try_wait()?.is_none() { + compiler.child.kill().await?; + } else { + compiler.child.wait().await?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs new file mode 100644 index 0000000000..86c4585092 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_sensor.rs @@ -0,0 +1,76 @@ +//! A callback gets its own interpreter; timeout and shutdown reap it before reporting a result. + +use std::future::Future; +use std::path::Path; +use std::time::Duration; + +use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::process::Child; + +use crate::native_commands::_helpers::execution::location_runtime_child; +use crate::native_commands::constants::{SENSOR_EVALUATION_SECONDS, SENSOR_RESULT_BYTES}; +use crate::native_commands::models::LocationServe; + +pub(super) async fn evaluate_location_sensor( + options: &LocationServe, + script: &Path, + evaluation: &SensorEvaluationRequest, + stopped: impl Future>, +) -> Result, ControlError> { + let request = + serde_json::to_vec(evaluation).map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut child = match location_runtime_child(options, script) { + Ok(child) => child, + Err(_) => return Ok(Some(CustomSensorOutcome::Failed)), + }; + tokio::pin!(stopped); + let outcome = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = tokio::time::timeout(Duration::from_secs(SENSOR_EVALUATION_SECONDS), exchange_result(&mut child, request)) => { + match result { + Ok(Ok((_, outcome))) => outcome, + Ok(Err(_)) => CustomSensorOutcome::Failed, + Err(_) => CustomSensorOutcome::TimedOut, + } + }, + }; + if child.try_wait()?.is_none() { + child.kill().await?; + } + Ok(Some(outcome)) +} + +/// A bounded reply is accepted only after its interpreter exits successfully. +async fn exchange_result( + child: &mut Child, + request: Vec, +) -> Result<(&mut Child, CustomSensorOutcome), ControlError> { + let mut stdin = child + .stdin + .take() + .ok_or_else(|| ControlError::Invalid("sensor input is unavailable".into()))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("sensor output is unavailable".into()))?; + stdin.write_all(&request).await?; + stdin.write_all(b"\n").await?; + stdin.shutdown().await?; + drop(stdin); + let mut bytes: Vec = Vec::new(); + let mut stdout = stdout.take(SENSOR_RESULT_BYTES + 1); + stdout.read_to_end(&mut bytes).await?; + if bytes.len() as u64 > SENSOR_RESULT_BYTES || !child.wait().await?.success() { + return Ok((child, CustomSensorOutcome::Failed)); + } + Ok(( + child, + serde_json::from_slice(&bytes).unwrap_or(CustomSensorOutcome::Failed), + )) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs new file mode 100644 index 0000000000..abf575eb77 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -0,0 +1,149 @@ +//! Named outbound work shares the watcher's generation boundary, never its interpreter. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use chrono::Utc; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::platform::models::{ + LocationWork, LocationWorkClient, LocationWorkOutcome, LocationWorkPoll, LocationWorkResult, +}; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::time::Instant; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::check_location_connection; +use crate::native_commands::_helpers::paths::location_inputs; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_location_sensor::evaluate_location_sensor; +use crate::native_commands::models::{LocationServe, LocationWatch}; + +/// Retry the exact completed result after uncertain delivery; never rerun it to obtain a new result. +pub(super) async fn run_location_work( + (options, script): (&LocationServe, &Path), + (client, poll, watch): (&LocationWorkClient, &LocationWorkPoll, &LocationWatch), + generation: Option, + stopped: impl Future>, +) -> Result>, ControlError> { + tokio::pin!(stopped); + let claim = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + result = client.poll(poll) => match result { + Ok(claim) => claim, + Err(error) => { + work_error(&error)?; + return Ok(Some(Vec::new())); + }, + }, + }; + let Some(claim) = claim else { + return Ok(Some(Vec::new())); + }; + let deadline = Instant::now() + + (claim.lease_expires_at - Utc::now()) + .to_std() + .unwrap_or_default(); + let mut files: Vec = Vec::new(); + let outcome = if generation == Some(claim.generation) + && location_inputs(&options.project_dir, &watch.rules, &watch.local_files) == watch.observed + { + let message = match &claim.request.work { + LocationWork::CheckConnection {} => "Checking requested location connection...\n", + LocationWork::EvaluateSensor { .. } => "Evaluating requested custom sensor...\n", + }; + drop(write_output(&[(Stream::Stderr, message)])); + let mut shutting_down = false; + let expires = async { + tokio::select! { + result = &mut stopped => { shutting_down = true; result }, + () = tokio::time::sleep_until(deadline) => Ok(()), + } + }; + let checked = match claim.request.work { + LocationWork::CheckConnection {} => check_location_connection(options, script, expires) + .await? + .map(|(healthy, files)| (LocationWorkOutcome::Connection { healthy }, files)), + LocationWork::EvaluateSensor { evaluation } => { + evaluate_location_sensor(options, script, &evaluation, expires) + .await? + .map(|outcome| { + ( + LocationWorkOutcome::SensorEvaluation { outcome }, + Vec::new(), + ) + }) + } + }; + let Some((outcome, local_files)) = checked else { + drop(write_output(&[( + Stream::Stderr, + "Location work stopped before completion; no result reported.\n", + )])); + return Ok((!shutting_down).then_some(files)); + }; + files = local_files; + let mut expected = watch.observed.clone(); + expected + .0 + .retain(|stamp| !files.contains(&options.project_dir.join(&stamp.relative_path))); + let mut excluded = watch.local_files.clone(); + excluded.extend(files.clone()); + if location_inputs(&options.project_dir, &watch.rules, &excluded) == expected { + outcome + } else { + LocationWorkOutcome::Superseded {} + } + } else { + LocationWorkOutcome::Superseded {} + }; + let result = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome, + }; + while Instant::now() < deadline { + let completed = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = tokio::time::sleep_until(deadline) => break, + result = client.complete(claim.request.request_id, &result) => result, + }; + match completed { + Ok(_) => { + drop(write_output(&[( + Stream::Stderr, + "Location work completed.\n", + )])); + return Ok(Some(files)); + } + Err(error) => { + work_error(&error)?; + if matches!(error, ClientError::Rejected(status) if status.is_client_error()) { + break; + } + } + } + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = tokio::time::sleep(Duration::from_secs(1)) => {}, + } + } + drop(write_output(&[( + Stream::Stderr, + "Location work acknowledgement expired; check the server result before retrying.\n", + )])); + Ok(Some(files)) +} + +fn work_error(error: &ClientError) -> Result<(), ControlError> { + if matches!(error, ClientError::Configuration(_)) + || matches!(error, ClientError::Rejected(status) if matches!(status.as_u16(), 401 | 403)) + { + return Err(ControlError::Invalid(error.to_string())); + } + drop(write_output(&[( + Stream::Stderr, + &format!("Location work request failed: {error}\n"), + )])); + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs new file mode 100644 index 0000000000..2580c2b194 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -0,0 +1,108 @@ +//! Direct database ownership authorizes bootstrap, not an untrusted HTTP request or supplied role. + +use serde_json::{Value, json}; +use sqlbuild_orchestration::credentials::main::create_location::create_location; +use sqlbuild_orchestration::credentials::main::create_principal::create_principal; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::declarations::main::change_trigger::change_trigger; +use sqlbuild_orchestration::declarations::models::TriggerChangeRequest; +use sqlbuild_orchestration::platform::main::location_work::location_work; +use sqlbuild_orchestration::platform::main::request_location_work::request_location_work; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; +use sqlbuild_orchestration::runs::main::request_rerun::request_rerun; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::control_admin; +use crate::native_commands::_helpers::execution::open_control_store; +use crate::native_commands::models::ControlAdmin; + +pub(crate) async fn run_admin(namespace: &Namespace) -> Result { + let action = control_admin(namespace)?; + let store = open_control_store(namespace).await?; + let context = AuditContext { + actor: "local-control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + match action { + ControlAdmin::CreateLocation(name) => { + let id = create_location(&store, &name, &context).await?; + Ok(json!({"location_id": id, "name": name})) + } + ControlAdmin::CreatePrincipal(name, kind) => { + let id = create_principal(&store, &context, (&name, kind)).await?; + Ok(json!({"principal_id": id, "name": name, "kind": kind})) + } + ControlAdmin::IssueToken(scope, expires_at) => { + let token = issue_token(&store, &context, scope, expires_at).await?; + Ok(json!(token)) + } + ControlAdmin::RevokeToken(token_id) => { + let revoked = revoke_token(&store, &context, token_id).await?; + Ok(json!({"token_id": token_id, "revoked": revoked})) + } + ControlAdmin::RequestJobRun(request) => { + let applied = request_job_run(&store, &context, &request).await?; + Ok(json!({"run_id": request.run_id, "applied": applied})) + } + ControlAdmin::Rerun(request) => { + let applied = request_rerun(&store, &context, &request).await?; + Ok(json!({"run_id": request.run_id, "applied": applied})) + } + ControlAdmin::RequestLocationWork(request) => { + let applied = request_location_work(&store, &context, &request).await?; + Ok(json!({"request_id": request.request_id, "applied": applied})) + } + ControlAdmin::LocationWork(request_id) => { + Ok(json!(location_work(&store, request_id).await?)) + } + ControlAdmin::CancelRun { + event_id, + run_id, + reason, + } => { + let applied = change_run( + &store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id, + run_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason, + }, + }, + ) + .await?; + Ok(json!({"event_id": event_id, "run_id": run_id, "applied": applied})) + } + ControlAdmin::ChangeTrigger { + event_id, + trigger_id, + change, + reason, + } => { + let applied = change_trigger( + &store, + &context, + &TriggerChangeRequest { + event_id, + trigger_id, + actor: context.actor.clone(), + reason, + change, + }, + ) + .await?; + Ok(json!({"event_id": event_id, "trigger_id": trigger_id, "applied": applied})) + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs new file mode 100644 index 0000000000..02e6928bd2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -0,0 +1,70 @@ +//! Supervise native API/scheduler roles together; a failed role stops its siblings. + +use sqlbuild_orchestration::api::main::serve_api::serve_api; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::SchedulerState; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{control_address, flag}; +use crate::native_commands::_helpers::execution::{ + open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, +}; +use crate::native_commands::_helpers::terminal::write_output; + +async fn stopped(mut shutdown: watch::Receiver) { + while !*shutdown.borrow_and_update() { + if shutdown.changed().await.is_err() { + break; + } + } +} + +pub(crate) async fn run_control( + namespace: &Namespace, + standalone: bool, +) -> Result<(), ControlError> { + let address = control_address(namespace, standalone)?; + let store = open_control_store(namespace).await?; + let workers = open_control_workers(namespace)?; + let (stop, shutdown) = watch::channel(false); + let (_changes, changes) = watch::channel(()); + let mut tasks = tokio::task::JoinSet::new(); + if let Some(address) = address { + drop(write_output(&[( + Stream::Stderr, + "Starting control API...\n", + )])); + let listener = tokio::net::TcpListener::bind(address).await?; + drop(write_output(&[( + Stream::Stderr, + &format!("Control API ready: {}\n", listener.local_addr()?), + )])); + tasks.spawn(serve_api( + store.clone(), + listener, + stopped(shutdown.clone()), + )); + start_platform_roles(&mut tasks, (&store, &shutdown))?; + } + if standalone || !flag(namespace, "control_no_scheduler") { + drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); + let scheduler_store = store.clone(); + tasks.spawn(async move { + run_scheduler( + &scheduler_store, + SchedulerState::new(Uuid::now_v7()), + (shutdown, changes), + workers, + ) + .await + }); + drop(write_output(&[( + Stream::Stderr, + "Scheduler loop started; leadership is lease-guarded\n", + )])); + } + supervise_control_roles(tasks, stop).await +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs new file mode 100644 index 0000000000..5a8082353a --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -0,0 +1,178 @@ +//! Coalesce source edits and retain the last good publication across isolated compiler failures. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use chrono::Utc; +use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; +use sqlbuild_orchestration::platform::models::{ + LocationHealth, LocationWorkClient, LocationWorkPoll, +}; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::sync::mpsc; +use tokio::time::{Instant, MissedTickBehavior}; +use uuid::Uuid; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::execution::{ + check_location_connection, compile_generation, location_generation, location_heartbeats, + report_location_health, shutdown_signal, +}; +use crate::native_commands::_helpers::paths::{ + exclude_location_databases, location_watch, observe_location_inputs, +}; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_location_compiler::run_location_compiler; +use crate::native_commands::main::_location_work::run_location_work; +use crate::native_commands::models::{LocationCompileRequest, LocationServe}; + +pub(super) async fn run_location( + options: LocationServe, + script: &Path, + bearer: &str, +) -> Result<(), ControlError> { + let session_id = Uuid::now_v7(); + let client = connect_location_work((&options.server_url, bearer), options.location_id) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let poll = LocationWorkPoll { + session_id, + target: options.target.clone(), + }; + let (requests, receiver) = mpsc::channel(1); + let serving = async { + tokio::select! { + result = watch_location(&options, script, (bearer, &client, &poll), requests) => result, + result = location_heartbeats(&options, bearer, session_id) => result, + } + }; + let (serving, compiler) = + tokio::join!(serving, run_location_compiler(&options, script, receiver)); + serving.and(compiler) +} + +async fn watch_location( + options: &LocationServe, + script: &Path, + (bearer, client, poll): (&str, &LocationWorkClient, &LocationWorkPoll), + requests: mpsc::Sender, +) -> Result<(), ControlError> { + let mut watch = location_watch(&options.project_dir); + let mut work_at = Instant::now() + Duration::from_secs(2); + let mut published_generation = None; + let mut clock = tokio::time::interval(Duration::from_millis(250)); + clock.set_missed_tick_behavior(MissedTickBehavior::Skip); + let stopped = shutdown_signal(); + tokio::pin!(stopped); + drop(write_output(&[( + Stream::Stderr, + "Starting code location; watching project inputs...\n", + )])); + loop { + tokio::select! { + result = &mut stopped => return result, + _ = clock.tick() => {}, + } + let ready; + (watch, ready) = observe_location_inputs(&options.project_dir, watch); + if !ready { + if Instant::now() >= work_at + && published_generation.is_some() + && watch.attempted.as_ref() == Some(&watch.observed) + { + let Some(files) = run_location_work( + (options, script), + (client, poll, &watch), + published_generation, + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + watch = exclude_location_databases(&options.project_dir, files, watch); + work_at = Instant::now() + Duration::from_secs(2); + } + continue; + } + published_generation = None; + let generation = tokio::select! { + result = &mut stopped => return result, + generation = location_generation(options, bearer) => generation?, + }; + let generation = match generation { + Some(generation) => generation, + None => { + watch.retry_at = Some(Instant::now() + Duration::from_secs(5)); + continue; + } + }; + drop(write_output(&[( + Stream::Stderr, + &format!("Compiling location generation {generation} in an isolated interpreter...\n"), + )])); + let Some((published, refresh_at)) = + compile_generation(&requests, generation, &watch.observed.0, stopped.as_mut()).await? + else { + return Ok(()); + }; + if published { + watch.retry_at = refresh_at.map(|deadline| { + Instant::now() + (deadline - Utc::now()).to_std().unwrap_or_default() + }); + published_generation = Some(generation); + let Some(files) = check_published_location( + options, + script, + (bearer, poll.session_id, generation), + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + watch = exclude_location_databases(&options.project_dir, files, watch); + } else { + watch.retry_at = Some(Instant::now() + Duration::from_secs(30)); + tokio::select! { + result = &mut stopped => return result, + () = report_location_health(options, bearer, poll.session_id, LocationHealth::CompileFailed { generation }) => {}, + } + drop(write_output(&[( + Stream::Stderr, + "Location reload failed; retaining last good snapshot. Retrying on change or in 30 seconds.\n", + )])); + } + } +} + +async fn check_published_location( + options: &LocationServe, + script: &Path, + (bearer, session_id, generation): (&str, Uuid, i64), + stopped: impl Future>, +) -> Result>, ControlError> { + drop(write_output(&[( + Stream::Stderr, + "Checking location connection with sqb debug...\n", + )])); + tokio::pin!(stopped); + let Some((connected, files)) = + check_location_connection(options, script, stopped.as_mut()).await? + else { + return Ok(None); + }; + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = report_location_health(options, bearer, session_id, LocationHealth::Connection { generation, healthy: connected }) => {}, + } + drop(write_output(&[( + Stream::Stderr, + if connected { + "Code location ready; connection check passed; watching for changes.\n" + } else { + "Code location published; connection check failed; watching for changes.\n" + }, + )])); + Ok(Some(files)) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index 034b2c7bcd..af7c252502 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,3 +1,12 @@ pub(super) mod _cost_report_options; +pub(super) mod _location_compiler; +pub(super) mod _location_sensor; +pub(super) mod _location_work; +pub(super) mod _run_admin; +pub(super) mod _run_control; pub(super) mod _run_cost; +pub(super) mod _run_location; +pub(crate) mod parse_native_invocation; +pub(crate) mod run_control_invocation; +pub(crate) mod run_location_invocation; pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs new file mode 100644 index 0000000000..8c2056d976 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/parse_native_invocation.rs @@ -0,0 +1,59 @@ +//! Parse once with the shared command table before resolving the optional Python CLI. + +use std::ffi::OsString; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; +use crate::native_commands::_helpers::decision::{review_json_failure, text}; +use crate::native_commands::_helpers::terminal::{ + finish_rejected_parse, stdout_supports_color, terminal_columns, +}; +use crate::native_commands::constants::{FALLBACK_COLUMNS, NO_COLOR_OPTION}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::NativeParse; + +pub(crate) fn parse_native_invocation(arguments: &[OsString]) -> Result { + let arguments: Vec = arguments + .iter() + .map(|argument| argument.to_str().map(str::to_owned)) + .collect::>() + .ok_or(PythonNeeded::Arguments)?; + let use_color = + !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); + let columns = terminal_columns(); + let needs_python_width = columns.is_err(); + let settings = ArgvSettings { + columns: columns.unwrap_or(FALLBACK_COLUMNS), + use_color, + }; + let mut host = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); + if needs_python_width + && !outcome.parsed.as_ref().is_ok_and(|namespace| { + text(namespace, "command").as_deref() == Some("location") + && text(namespace, "location_command").as_deref() == Some("serve") + }) + { + return Err(PythonNeeded::Arguments); + } + if host.needs_python { + return Err(PythonNeeded::Arguments); + } + let namespace: Namespace = match outcome.parsed { + Err(_) if review_json_failure(&arguments, &outcome) => return Err(PythonNeeded::Arguments), + Err(code) => { + return Ok(NativeParse::Exit(finish_rejected_parse( + &outcome.writes, + code, + ))); + } + Ok(namespace) if outcome.writes.is_empty() => namespace, + Ok(_) => return Err(PythonNeeded::Arguments), + }; + Ok(NativeParse::Invocation { + namespace, + use_color, + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs new file mode 100644 index 0000000000..6ff78bf47b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_control_invocation.rs @@ -0,0 +1,90 @@ +//! Control roles are served before Python/package discovery, including their JSON contract. + +use sqlbuild_orchestration::api::main::openapi_document::openapi_document; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::{flag, text}; +use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; +use crate::native_commands::constants::{ADMIN_COMMAND, SCHEDULER_COMMAND, SERVER_COMMAND}; +use crate::native_commands::main::_run_admin::run_admin; +use crate::native_commands::main::_run_control::run_control; + +pub(crate) fn run_control_invocation(namespace: &Namespace) -> Option { + let command = text(namespace, "command")?; + if !matches!( + command.as_str(), + SERVER_COMMAND | SCHEDULER_COMMAND | ADMIN_COMMAND + ) { + return None; + } + if flag(namespace, "control_openapi") { + return Some(match serde_json::to_string(&openapi_document()) { + Ok(document) => { + let document = document + "\n"; + let lost = write_output(&[(Stream::Stdout, document.as_str())]); + finish_lost_stdout(lost.as_ref(), 0) + } + Err(_) => { + drop(write_output(&[( + Stream::Stderr, + "Control API contract generation failed\n", + )])); + 1 + } + }); + } + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control service failed to start: {error}\n"), + )])); + return Some(1); + } + }; + let admin = command == ADMIN_COMMAND; + let role = if admin { "administration" } else { "service" }; + let result = runtime.block_on(async { + if admin { + run_admin(namespace).await.map(Some) + } else { + run_control(namespace, command == SCHEDULER_COMMAND) + .await + .map(|()| None) + } + }); + Some(match result { + Ok(output) => { + let lost = + output.and_then(|value| write_output(&[(Stream::Stdout, &format!("{value}\n"))])); + if admin && lost.is_some() { + drop(write_output(&[( + Stream::Stderr, + "Control administration JSON result delivery failed; changes may already be committed\n", + )])); + return Some(1); + } + let exit_code = finish_lost_stdout(lost.as_ref(), 0); + if exit_code == 0 { + let message = if admin { + "Control administration completed successfully\n" + } else { + "Control service stopped successfully\n" + }; + drop(write_output(&[(Stream::Stderr, message)])); + } + exit_code + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Control {role} failed: {error}\n"), + )])); + 1 + } + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs new file mode 100644 index 0000000000..b9869af6a9 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs @@ -0,0 +1,48 @@ +//! A location supervisor stays native while each compilation uses the installed project interpreter. + +use std::path::Path; + +use crate::argv::models::{Namespace, Stream}; +use crate::native_commands::_helpers::decision::text; +use crate::native_commands::_helpers::execution::location_options; +use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_run_location::run_location; + +pub(crate) fn run_location_invocation(namespace: &Namespace, script: &Path) -> Option { + if text(namespace, "command").as_deref() != Some("location") + || text(namespace, "location_command").as_deref() != Some("serve") + { + return None; + } + let result = (|| { + let options = location_options(namespace)?; + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + std::io::Error::other( + "SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential", + ) + })?; + if bearer.trim().is_empty() { + return Err(std::io::Error::other("SQLBUILD_LOCATION_TOKEN must not be blank").into()); + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + runtime.block_on(run_location(options, script, &bearer)) + })(); + Some(match result { + Ok(()) => { + drop(write_output(&[( + Stream::Stderr, + "Code location stopped successfully\n", + )])); + 0 + } + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("Code location failed: {error}\n"), + )])); + 1 + } + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs index 40360a7de8..c92474afef 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -1,50 +1,19 @@ //! Serve one invocation without Python when its parse and project facts allow it. -use std::ffi::OsString; use std::path::Path; -use crate::argv::main::parse_cli_arguments::parse_cli_arguments; -use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; -use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; -use crate::native_commands::_helpers::decision::{native_invocation, review_json_failure}; +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::native_invocation; use crate::native_commands::_helpers::execution::run_command; -use crate::native_commands::_helpers::terminal::{ - finish_rejected_parse, stdout_supports_color, terminal_columns, -}; -use crate::native_commands::constants::NO_COLOR_OPTION; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::NativeInvocation; /// The exit status of an invocation `sqb` served, or why the same arguments go to Python. pub fn run_native_invocation( - arguments: &[OsString], + namespace: &Namespace, python_cli: &Path, + use_color: bool, ) -> Result { - let arguments: Vec = arguments - .iter() - .map(|argument| argument.to_str().map(str::to_owned)) - .collect::>() - .ok_or(PythonNeeded::Arguments)?; - let use_color: bool = - !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); - let settings: ArgvSettings = ArgvSettings { - columns: terminal_columns()?, - use_color, - }; - let mut host: NativeArgvHost = NativeArgvHost::default(); - let outcome: ArgvOutcome = - parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); - if host.needs_python { - return Err(PythonNeeded::Arguments); - } - let namespace: Namespace = match outcome.parsed { - Err(_) if review_json_failure(&arguments, &outcome) => { - return Err(PythonNeeded::Arguments); - } - Err(code) => return Ok(finish_rejected_parse(&outcome.writes, code)), - Ok(namespace) if outcome.writes.is_empty() => namespace, - Ok(_) => return Err(PythonNeeded::Arguments), - }; - let invocation: NativeInvocation = native_invocation(&namespace)?; + let invocation: NativeInvocation = native_invocation(namespace)?; run_command(&invocation, python_cli, use_color) } diff --git a/crates/sqlbuild-cli/src/native_commands/mod.rs b/crates/sqlbuild-cli/src/native_commands/mod.rs index d99ca200f2..5ac20489d5 100644 --- a/crates/sqlbuild-cli/src/native_commands/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod constants; pub mod errors; pub mod main; pub(crate) mod models; +pub(crate) mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 93db875a33..b16504cefa 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -2,6 +2,50 @@ use std::path::PathBuf; +use chrono::{DateTime, Utc}; +use sqlbuild_cache::project_snapshot::models::{PathStamp, SnapshotRules}; +use sqlbuild_orchestration::credentials::models::{CredentialScope, PrincipalKind}; +use sqlbuild_orchestration::declarations::models::TriggerChange; +use sqlbuild_orchestration::platform::models::LocationWorkRequest; +use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; +use uuid::Uuid; + +use crate::argv::models::Namespace; +use crate::native_commands::types::LocationPublication; + +/// Privileged local operations require direct ownership of the control database. +pub(crate) enum ControlAdmin { + CreateLocation(String), + CreatePrincipal(String, PrincipalKind), + IssueToken(CredentialScope, DateTime), + RevokeToken(Uuid), + RequestJobRun(JobRunRequest), + Rerun(RerunRequest), + RequestLocationWork(LocationWorkRequest), + LocationWork(Uuid), + CancelRun { + event_id: Uuid, + run_id: Uuid, + reason: String, + }, + ChangeTrigger { + event_id: Uuid, + trigger_id: Uuid, + change: TriggerChange, + reason: String, + }, +} + +/// Parsing can finish an invocation before package discovery or any Python handoff. +#[derive(Debug)] +pub(crate) enum NativeParse { + Exit(i32), + Invocation { + namespace: Namespace, + use_color: bool, + }, +} + /// What `sqb skills` installs. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct SkillsOptions { @@ -88,6 +132,44 @@ pub(crate) struct ProjectLocation { pub(crate) resolved: PathBuf, } +/// A long-running location owns no control-store credentials or executable user objects. +#[derive(Debug)] +pub(crate) struct LocationServe { + pub(crate) project_dir: PathBuf, + pub(crate) server_url: String, + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) image_id: Option, +} + +/// Retained source observations survive compilation; warehouse exclusions never replace them. +#[derive(Debug)] +pub(crate) struct LocationWatch { + pub(crate) rules: SnapshotRules, + pub(crate) local_files: Vec, + pub(crate) observed: (Vec, Option), + pub(crate) attempted: Option<(Vec, Option)>, + pub(crate) changed_at: tokio::time::Instant, + pub(crate) retry_at: Option, +} + +/// A bounded supervisor request; dropping the reply cancels and reaps an active compiler. +#[derive(Debug)] +pub(crate) struct LocationCompileRequest { + pub(crate) generation: i64, + pub(crate) python_sources: Vec, + pub(crate) reply: tokio::sync::oneshot::Sender, String>>, +} + +/// Only this task owns the isolated interpreter and its private stdio protocol. +#[derive(Debug)] +pub(crate) struct LocationCompiler { + pub(crate) child: tokio::process::Child, + pub(crate) stdin: tokio::process::ChildStdin, + pub(crate) stdout: tokio::io::BufReader, + pub(crate) python_sources: Vec, +} + /// What Python's buffered stdout does with output its reader no longer takes. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LostStdout { diff --git a/crates/sqlbuild-cli/src/native_commands/types.rs b/crates/sqlbuild-cli/src/native_commands/types.rs new file mode 100644 index 0000000000..438f742268 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/types.rs @@ -0,0 +1,7 @@ +//! Private interpreter protocol shapes owned by the location supervisor. + +use chrono::{DateTime, Utc}; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; + +/// Publication metadata and the earliest calendar refresh deadline. +pub(crate) type LocationPublication = (SnapshotRecord, Option>); diff --git a/crates/sqlbuild-cli/src/workers/_helpers/child.rs b/crates/sqlbuild-cli/src/workers/_helpers/child.rs new file mode 100644 index 0000000000..75d44e336f --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/child.rs @@ -0,0 +1,63 @@ +//! Launchers share physical exit observation and termination, not durable lifecycle authority. + +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::supervisor::{exit_code, terminate}; +use crate::workers::models::{WorkerChild, WorkerExit}; + +impl WorkerExit { + pub(in crate::workers) const fn code(self) -> Option { + match self { + Self::Code(code) => Some(code), + Self::Absent => None, + } + } +} + +impl WorkerChild { + pub(in crate::workers) async fn try_wait( + &mut self, + ) -> Result, ControlError> { + match self { + Self::Local(child) => Ok(child + .try_wait()? + .map(|exit| WorkerExit::Code(exit_code(exit)))), + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.try_wait(), + Self::Docker(child) => child.try_wait().await, + } + } + + pub(in crate::workers) async fn wait(&mut self) -> Result { + if let Self::Local(child) = self { + return Ok(WorkerExit::Code(exit_code(child.wait().await?))); + } + loop { + if let Some(exit) = self.try_wait().await? { + return Ok(exit); + } + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + } + + pub(in crate::workers) async fn terminate(&mut self) -> Result<(), ControlError> { + match self { + Self::Local(child) => terminate(child), + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::TERM), + Self::Docker(child) => child.signal("SIGTERM").await, + } + } + + pub(in crate::workers) async fn kill(&mut self) -> Result<(), ControlError> { + match self { + Self::Local(child) => { + child.start_kill()?; + Ok(()) + } + #[cfg(target_os = "linux")] + Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::KILL), + Self::Docker(child) => child.signal("SIGKILL").await, + } + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs new file mode 100644 index 0000000000..0027b59f29 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/configuration.rs @@ -0,0 +1,91 @@ +//! Reject ambiguous or nonexistent installation-owned launcher scopes before claiming any run. + +use std::collections::BTreeSet; + +use sqlbuild_orchestration::store::errors::ControlError; +use uuid::Uuid; + +use crate::workers::models::{Launcher, LauncherScope, Launchers}; + +impl<'a> Launcher<'a> { + pub(in crate::workers) fn scope(self) -> LauncherScope<'a> { + match self { + Self::Local(location) => LauncherScope { + location_id: location.location_id, + target: &location.target, + server_url: &location.server_url, + max_workers: location.max_workers, + }, + Self::Docker(location) => LauncherScope { + location_id: location.location_id, + target: &location.target, + server_url: &location.server_url, + max_workers: location.max_workers, + }, + } + } +} + +impl Launchers { + pub(in crate::workers) fn locations(&self) -> impl Iterator> { + self.local + .iter() + .map(Launcher::Local) + .chain(self.docker.iter().map(Launcher::Docker)) + } + + pub(in crate::workers) fn dispatch_scopes(&self) -> BTreeSet<(Uuid, String)> { + self.locations() + .map(|location| { + let scope = location.scope(); + (scope.location_id, scope.target.to_owned()) + }) + .collect() + } +} + +pub(in crate::workers) fn read_configuration( + path: Option<&str>, + scheduler_enabled: bool, +) -> Result { + let configuration: Launchers = match path { + Some(path) => serde_json::from_slice(&std::fs::read(path)?) + .map_err(|error| ControlError::Invalid(format!("launcher configuration: {error}")))?, + None => Launchers { + local: Vec::new(), + docker: Vec::new(), + concurrency_limits: Default::default(), + }, + }; + if !scheduler_enabled && configuration.locations().next().is_some() { + return Err(ControlError::Invalid( + "launchers require the embedded scheduler".into(), + )); + } + let mut scopes: BTreeSet<(Uuid, &str)> = BTreeSet::new(); + for location in &configuration.local { + if !location.project_dir.is_absolute() + || !location.project_dir.is_dir() + || !location.executable.is_absolute() + || !location.executable.is_file() + { + return Err(ControlError::Invalid("local launchers require unique location/target scopes and existing absolute project/executable paths".into())); + } + } + for location in configuration.locations() { + let scope = location.scope(); + if scope.target.trim().is_empty() || !scopes.insert((scope.location_id, scope.target)) { + return Err(ControlError::Invalid( + "launchers require unique location/target scopes".into(), + )); + } + if let Launcher::Docker(location) = location + && (!location.socket.is_absolute() + || location.location_container.trim().is_empty() + || location.executable.trim().is_empty()) + { + return Err(ControlError::Invalid("Docker launchers require an absolute socket path, location container and executable".into())); + } + } + Ok(configuration) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/docker.rs b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs new file mode 100644 index 0000000000..7b0d5c9633 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/docker.rs @@ -0,0 +1,205 @@ +//! The privileged role talks to a local Docker daemon; workers never receive its socket. + +use std::path::Path; + +use reqwest::{Method, StatusCode}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::constants::IMAGE_ID_HEX_LENGTH; +use crate::workers::models::{DockerLauncher, DockerWorker, Launcher, WorkerExit}; + +impl DockerWorker { + pub(in crate::workers) fn prepare( + location: &DockerLauncher, + claim: &RunClaim, + ) -> Result { + let image_id = claim.request.image_id.as_deref().unwrap_or_default(); + match image_id.strip_prefix("sha256:") { + Some(digest) + if digest.len() == IMAGE_ID_HEX_LENGTH + && digest.as_bytes().iter().all(u8::is_ascii_hexdigit) => {} + _ => { + return Err(ControlError::Invalid( + "Docker workers require a snapshot pinned to a full sha256 image ID".into(), + )); + } + } + Ok(Self { + client: docker_client(&location.socket)?, + name: format!( + "sqlbuild-{}-{}", + claim.request.location_id, claim.request.run_id + ), + claim_id: claim.claim_id, + image_id: image_id.into(), + }) + } + + pub(in crate::workers) async fn launch( + &self, + location: &DockerLauncher, + claim: &RunClaim, + bearer: &str, + control_variable: &str, + ) -> Result<(), ControlError> { + let parent: Value = self + .request( + Method::GET, + &format!("containers/{}/json", location.location_container), + None, + ) + .await? + .error_for_status() + .map_err(docker_error)? + .json() + .await + .map_err(docker_error)?; + let project = parent["Config"]["WorkingDir"] + .as_str() + .filter(|path| path.starts_with('/')) + .ok_or_else(|| { + ControlError::Invalid( + "location container requires an absolute working directory".into(), + ) + })?; + let mut command = arguments(&Launcher::Docker(location).scope(), claim); + command.push(project.into()); + command.extend(["--image-id".into(), self.image_id.clone()]); + let mut variables: Vec = parent["Config"]["Env"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|entry| !excluded(entry.split('=').next().unwrap_or(entry), control_variable)) + .map(str::to_owned) + .collect(); + variables.extend( + environment(claim, bearer) + .into_iter() + .map(|(key, value)| format!("{key}={value}")), + ); + let body = json!({ + "Image": self.image_id, "Entrypoint": [location.executable], "Cmd": command, + "WorkingDir": project, "User": parent["Config"]["User"], "Env": variables, + "Labels": {"io.sqlbuild.claim-id": self.claim_id.to_string()}, + "HostConfig": { + "NetworkMode": format!("container:{}", location.location_container), + "VolumesFrom": [location.location_container], + "Memory": parent["HostConfig"]["Memory"], "MemorySwap": parent["HostConfig"]["MemorySwap"], + "NanoCpus": parent["HostConfig"]["NanoCpus"], "PidsLimit": parent["HostConfig"]["PidsLimit"], + "ReadonlyRootfs": parent["HostConfig"]["ReadonlyRootfs"], + }, + }); + self.request( + Method::POST, + &format!("containers/create?name={}", self.name), + Some(&body), + ) + .await? + .error_for_status() + .map_err(docker_error)?; + self.request( + Method::POST, + &format!("containers/{}/start", self.name), + None, + ) + .await? + .error_for_status() + .map_err(docker_error)?; + Ok(()) + } + + pub(in crate::workers) async fn try_wait(&self) -> Result, ControlError> { + let response = self + .request(Method::GET, &format!("containers/{}/json", self.name), None) + .await?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(Some(WorkerExit::Absent)); + } + let container: Value = response + .error_for_status() + .map_err(docker_error)? + .json() + .await + .map_err(docker_error)?; + if container["Config"]["Labels"]["io.sqlbuild.claim-id"] != self.claim_id.to_string() + || container["Image"] != self.image_id + { + return Err(ControlError::Invalid( + "Docker container does not belong to the retained claim and image".into(), + )); + } + if container["State"]["Running"].as_bool() == Some(true) { + return Ok(None); + } + if container["State"]["Status"].as_str() == Some("created") { + return Ok(Some(WorkerExit::Absent)); + } + let code = container["State"]["ExitCode"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("Docker container has no exit evidence".into()))?; + Ok(Some(WorkerExit::Code(code))) + } + + pub(in crate::workers) async fn signal(&self, signal: &str) -> Result<(), ControlError> { + if self.try_wait().await?.is_some() { + return Ok(()); + } + let response = self + .request( + Method::POST, + &format!("containers/{}/kill?signal={signal}", self.name), + None, + ) + .await?; + if !matches!( + response.status(), + StatusCode::NOT_FOUND | StatusCode::CONFLICT + ) { + response.error_for_status().map_err(docker_error)?; + } + Ok(()) + } + + async fn request( + &self, + method: Method, + path: &str, + body: Option<&Value>, + ) -> Result { + let request = self + .client + .request(method, format!("http://localhost/{path}")); + match body { + Some(body) => request.json(body), + None => request, + } + .send() + .await + .map_err(docker_error) + } +} + +fn docker_error(error: reqwest::Error) -> ControlError { + ControlError::Invalid(format!("Docker daemon request failed: {error}")) +} + +#[cfg(unix)] +fn docker_client(socket: &Path) -> Result { + reqwest::Client::builder() + .no_proxy() + .unix_socket(socket) + .timeout(std::time::Duration::from_secs(5)) + .build() + .map_err(docker_error) +} + +#[cfg(not(unix))] +fn docker_client(_socket: &Path) -> Result { + Err(ControlError::Invalid( + "Docker launchers require a Unix socket".into(), + )) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs new file mode 100644 index 0000000000..a951cc74d6 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/invocation.rs @@ -0,0 +1,49 @@ +//! Both launchers hand off the same scoped invocation and exclude privileged credentials. + +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; +use sqlbuild_orchestration::runs::models::RunClaim; + +use crate::workers::models::LauncherScope; + +pub(in crate::workers) fn arguments(scope: &LauncherScope<'_>, claim: &RunClaim) -> Vec { + [ + "location".into(), + "execute".into(), + "--server".into(), + scope.server_url.to_owned(), + "--location-id".into(), + scope.location_id.to_string(), + "--target".into(), + scope.target.to_owned(), + "--run-id".into(), + claim.request.run_id.to_string(), + "--claim-id".into(), + claim.claim_id.to_string(), + "--project-dir".into(), + ] + .into() +} + +pub(in crate::workers) fn excluded(name: &str, control_variable: &str) -> bool { + name == control_variable + || name == PLATFORM_WEBHOOK_VARIABLE + || matches!( + name, + "SQLBUILD_CONTROL_DATABASE_URL" | "SQLBUILD_LOCATION_TOKEN" | "SQLBUILD_RUN_TOKEN" + ) + || name == INVOCATION_CONTEXT_ENV +} + +pub(in crate::workers) fn environment( + claim: &RunClaim, + bearer: &str, +) -> [(&'static str, String); 2] { + [ + ("SQLBUILD_RUN_TOKEN", bearer.into()), + (INVOCATION_CONTEXT_ENV, serde_json::json!({ + "run_id": claim.request.run_id, + "integration": {"name": "sqlbuild", "run_id": claim.request.run_id, "job_name": claim.request.job_key}, + }).to_string()), + ] +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/local.rs b/crates/sqlbuild-cli/src/workers/_helpers/local.rs new file mode 100644 index 0000000000..82a1c4fba2 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/local.rs @@ -0,0 +1,128 @@ +//! Linux dev workers inherit a claim lock before exec; recovery never guesses from a stale PID. + +use std::fs::OpenOptions; +use std::os::unix::fs::MetadataExt; + +use command_fds::{CommandFdExt, FdMapping}; +use rustix::fs::{FlockOperation, flock}; +use rustix::process::{Pid, PidfdFlags, Signal, pidfd_open, pidfd_send_signal}; +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; +use sqlbuild_orchestration::runs::models::RunClaim; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::process::Command; + +use crate::workers::constants::LOCAL_CLAIM_FD; +use crate::workers::models::{LocalLauncher, LocalWorker, WorkerExit}; + +impl LocalWorker { + pub(in crate::workers) fn open( + location: &LocalLauncher, + claim: &RunClaim, + ) -> Result { + let directory = location.project_dir.join(".sqlbuild/workers"); + std::fs::create_dir_all(&directory)?; + let lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(directory.join(format!("{}.lock", claim.claim_id)))?; + Ok(Self { + lock, + run_id: claim.request.run_id, + stopped: false, + }) + } + + /// Keep the acquired lock until terminal publication, so a delayed spawn cannot start afterward. + pub(in crate::workers) fn try_wait(&mut self) -> Result, ControlError> { + if !self.stopped { + match flock(&self.lock, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => self.stopped = true, + Err(rustix::io::Errno::WOULDBLOCK) => return Ok(None), + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + Ok(Some(WorkerExit::Absent)) + } + + /// Only a kernel handle whose current descriptor and invocation match this claim can be signalled. + pub(in crate::workers) fn signal(&self, signal: Signal) -> Result<(), ControlError> { + let expected = self.lock.metadata()?; + for entry in std::fs::read_dir("/proc")? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let raw = match name.parse::() { + Ok(raw) => raw, + Err(_) => continue, + }; + let Some(pid) = Pid::from_raw(raw) else { + continue; + }; + let descriptor = entry.path().join(format!("fd/{LOCAL_CLAIM_FD}")); + if !same_file(&descriptor, &expected) { + continue; + } + let handle = match pidfd_open(pid, PidfdFlags::empty()) { + Ok(handle) => handle, + Err(rustix::io::Errno::SRCH) => continue, + Err(error) => return Err(std::io::Error::from(error).into()), + }; + if !same_file(&descriptor, &expected) || !self.owns_invocation(&entry.path())? { + continue; + } + match pidfd_send_signal(handle, signal) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + Ok(()) + } + + fn owns_invocation(&self, process: &std::path::Path) -> Result { + let environment = match std::fs::read(process.join("environ")) { + Ok(environment) => environment, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + }; + let prefix = format!("{INVOCATION_CONTEXT_ENV}="); + for variable in environment.split(|byte| *byte == 0) { + if let Some(context) = variable.strip_prefix(prefix.as_bytes()) { + let context: serde_json::Value = + serde_json::from_slice(context).map_err(|error| { + ControlError::Invalid(format!( + "invalid local worker invocation context: {error}" + )) + })?; + return Ok(context["run_id"].as_str() == Some(&self.run_id.to_string())); + } + } + Ok(false) + } +} + +fn same_file(path: &std::path::Path, expected: &std::fs::Metadata) -> bool { + std::fs::metadata(path) + .is_ok_and(|actual| actual.dev() == expected.dev() && actual.ino() == expected.ino()) +} + +/// The child inherits the lock even before Python starts or a worker handshake can succeed. +pub(in crate::workers) fn inherit_claim( + command: &mut Command, + location: &LocalLauncher, + claim: &RunClaim, +) -> Result<(), ControlError> { + let owner = LocalWorker::open(location, claim)?; + flock(&owner.lock, FlockOperation::NonBlockingLockExclusive).map_err(std::io::Error::from)?; + command + .as_std_mut() + .fd_mappings(vec![FdMapping { + parent_fd: owner.lock.into(), + child_fd: LOCAL_CLAIM_FD, + }]) + .map_err(|error| ControlError::Invalid(format!("local claim descriptor: {error}")))?; + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/mod.rs b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs new file mode 100644 index 0000000000..a0ead33080 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/mod.rs @@ -0,0 +1,9 @@ +mod child; +pub(super) mod configuration; +mod docker; +mod invocation; +#[cfg(target_os = "linux")] +mod local; +mod recovery; +mod shutdown; +mod supervisor; diff --git a/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs new file mode 100644 index 0000000000..f439a58dd6 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/recovery.rs @@ -0,0 +1,86 @@ +//! A new leader resumes supervising pinned containers without relaunching their execution. + +use std::collections::BTreeSet; +use std::io::Write; + +use sqlbuild_orchestration::runs::main::list_run_claims::list_run_claims; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::workers::models::{ClaimedWorker, DockerWorker, Launcher, WorkerChild, Workers}; + +impl Workers { + pub(in crate::workers) async fn recover( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + if self.recovered_epoch == Some(lease.epoch) { + return Ok(()); + } + let mut known: BTreeSet<_> = self + .active + .iter() + .map(|worker| worker.claim.claim_id) + .collect(); + for location in self.configuration.locations() { + #[cfg(not(target_os = "linux"))] + if matches!(location, Launcher::Local(_)) { + continue; + } + let scope = location.scope(); + let mut after = None; + loop { + renew_lease(store, lease).await?; + let claims = + list_run_claims(store, (scope.location_id, scope.target), after).await?; + let Some(last) = claims.last() else { + break; + }; + after = Some(last.request.run_id); + for claim in claims { + if !known.insert(claim.claim_id) { + continue; + } + let child = match location { + #[cfg(target_os = "linux")] + Launcher::Local(location) => Some(WorkerChild::RecoveredLocal( + crate::workers::models::LocalWorker::open(location, &claim)?, + )), + #[cfg(not(target_os = "linux"))] + Launcher::Local(_) => continue, + Launcher::Docker(location) => match DockerWorker::prepare(location, &claim) + { + Ok(child) => Some(WorkerChild::Docker(child)), + Err(ControlError::Invalid(_)) => None, + Err(error) => return Err(error), + }, + }; + drop(writeln!( + std::io::stderr().lock(), + "Worker supervision recovered for run {}", + claim.request.run_id + )); + let rejected = child.is_none(); + let index = self.active.len(); + self.active.push(ClaimedWorker { + claim, + child, + stopping: None, + exit: None, + launch_failure: rejected.then_some("docker_configuration_rejected"), + }); + if rejected { + self.active[index] + .publish_launch_failure(store, lease) + .await?; + } + } + } + } + self.recovered_epoch = Some(lease.epoch); + Ok(()) + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs new file mode 100644 index 0000000000..7eafd6f876 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/shutdown.rs @@ -0,0 +1,189 @@ +//! Keep leadership alive while reaping children, then release only successfully reported claims. + +use std::io::Write; +use std::time::{Duration, Instant}; + +use sqlbuild_orchestration::runs::main::observe_run::observe_run; +use sqlbuild_orchestration::runs::models::{RunChange, RunOutcome}; +use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; + +use crate::workers::_helpers::supervisor::{publish, terminal}; +use crate::workers::models::{ClaimedWorker, WorkerChild, WorkerExit, Workers}; + +impl ClaimedWorker { + async fn reap( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + let Some(child) = self.child.as_mut() else { + return Ok(()); + }; + if self.exit.is_some() { + return Ok(()); + } + let mut killed = false; + if self.stopping.is_none() { + if child.terminate().await.is_err() { + child.kill().await?; + killed = true; + } + self.stopping = Some(Instant::now()); + } + let deadline = self.stopping.unwrap_or_else(Instant::now) + Duration::from_secs(60); + let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + loop { + tokio::select! { + exit = child.wait() => { + self.exit = Some(exit?); + return Ok(()); + } + _ = tokio::time::sleep_until(deadline.into()), if !killed => { + child.kill().await?; + killed = true; + } + _ = renew.tick(), if lease.is_some() => { + if let Some(lease) = lease + && renew_lease(store, lease).await.is_err() + { + drop(writeln!(std::io::stderr().lock(), "Scheduler lease renewal failed during worker shutdown")); + } + } + } + } + } + + async fn finish_shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.reap(store, lease).await?; + let state = observe_run(store, self.claim.request.run_id).await?; + if !terminal(state.status) { + let lease = lease.ok_or(ControlError::Fenced)?; + let reason = if self.child.is_none() { + "worker launch preparation failed before a child started" + } else { + "scheduler confirmed its worker stopped or absent" + }; + publish( + store, + lease, + &self.claim, + ( + "local-shutdown", + RunChange::Finished { + outcome: if state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else { + RunOutcome::Failed + }, + reason: reason.into(), + exit_code: self.exit.and_then(WorkerExit::code), + }, + ), + ) + .await?; + } + Ok(()) + } +} + +impl Workers { + pub(in crate::workers) async fn shutdown_workers( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + let mut retrying = false; + let mut renew = tokio::time::interval(Duration::from_secs(RENEW_INTERVAL_SECONDS)); + let mut renewed = Instant::now(); + while !self.active.is_empty() { + let mut failure = None; + for index in (0..self.active.len()).rev() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) + && let Some(lease) = lease + { + if let Err(error) = renew_lease(store, lease).await { + failure = Some(error); + } + renewed = Instant::now(); + } + if self.active[index].launch_failure.is_some() { + let reported = match lease { + Some(lease) => { + self.active[index] + .publish_launch_failure(store, lease) + .await + } + None => Err(ControlError::Fenced), + }; + if let Err(error) = reported { + if matches!(error, ControlError::Fenced) { + failure = Some(error); + } else { + failure.get_or_insert(error); + } + continue; + } + } + if let Some(WorkerChild::Docker(child)) = &self.active[index].child { + match child.try_wait().await { + Ok(None) => { + self.active.swap_remove(index); + continue; + } + Ok(exit) => self.active[index].exit = exit, + Err(error) => { + failure.get_or_insert(error); + continue; + } + } + } + match self.active[index].finish_shutdown(store, lease).await { + Ok(()) => { + self.active.swap_remove(index); + } + Err(ControlError::Fenced) => failure = Some(ControlError::Fenced), + Err(error) => { + failure.get_or_insert(error); + } + } + } + if matches!(failure, Some(ControlError::Fenced)) { + return Err(ControlError::Fenced); + } + if failure.is_some() { + if !retrying { + drop(writeln!( + std::io::stderr().lock(), + "Retrying worker shutdown publication or physical observation; claims remain owned" + )); + retrying = true; + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(1)) => {}, + _ = renew.tick(), if lease.is_some() => { + if let Some(lease) = lease + && matches!(renew_lease(store, lease).await, Err(ControlError::Fenced)) + { + return Err(ControlError::Fenced); + } + } + } + } + } + if retrying { + drop(writeln!( + std::io::stderr().lock(), + "Worker shutdown recovered; stopped claims released and live containers remain recoverable" + )); + } + Ok(()) + } +} diff --git a/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs new file mode 100644 index 0000000000..ab053c0d10 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/_helpers/supervisor.rs @@ -0,0 +1,409 @@ +//! One privileged owner supervises both launchers; workers receive only claim-scoped HTTP. + +use std::io::Write; +use std::process::{ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +use chrono::{TimeDelta, Utc}; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope}; +use sqlbuild_orchestration::runs::main::advance_groups::advance_groups; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::main::observe_run::observe_run; +use sqlbuild_orchestration::runs::models::{ + RunAuthority, RunChange, RunChangeRequest, RunClaim, RunDispatch, RunOutcome, RunStatus, +}; +use sqlbuild_orchestration::scheduler::constants::RENEW_INTERVAL_SECONDS; +use sqlbuild_orchestration::scheduler::main::renew_lease::renew_lease; +use sqlbuild_orchestration::scheduler::models::SchedulerLease; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; +use tokio::process::{Child, Command}; +use uuid::Uuid; + +use crate::workers::_helpers::invocation::{arguments, environment, excluded}; +use crate::workers::models::{ + ClaimedWorker, DockerWorker, Launcher, LocalLauncher, WorkerChild, WorkerExit, Workers, +}; + +impl sqlbuild_orchestration::scheduler::types::RunLauncher for Workers { + async fn poll( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + if let Some(lease) = lease { + let eligible = self.configuration.dispatch_scopes(); + for location in self.configuration.locations() { + let scope = location.scope(); + advance_groups( + store, + lease, + ( + &self.configuration.concurrency_limits, + RunDispatch { + scope: Some((scope.location_id, scope.target)), + eligible: Some(&eligible), + }, + ), + ) + .await?; + } + self.recover(store, lease).await?; + } + let mut renewed = Instant::now(); + for index in (0..self.active.len()).rev() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) + && let Some(lease) = lease + { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } + let worker = &mut self.active[index]; + let mut state = observe_run(store, worker.claim.request.run_id).await?; + if let Some(child) = worker.child.as_mut() { + worker.exit = worker.exit.or(child.try_wait().await?); + } + if worker.exit.is_none() + && let Some(child) = worker.child.as_mut() + { + let now = state.observed_at; + let not_started = matches!(state.status, RunStatus::Queued | RunStatus::Launched) + && state + .claimed_at + .is_some_and(|claimed| now - claimed > TimeDelta::minutes(2)); + let silent = state + .last_heartbeat_at + .is_some_and(|last| now - last > TimeDelta::minutes(10)); + if (state.cancel_requested_at.is_some() || not_started || silent) + && worker.stopping.is_none() + { + child.terminate().await?; + worker.stopping = Some(Instant::now()); + } + if worker + .stopping + .is_some_and(|at| at.elapsed() >= Duration::from_secs(60)) + { + child.kill().await?; + worker.exit = child.try_wait().await?; + state = observe_run(store, worker.claim.request.run_id).await?; + } + } + if worker.exit.is_some() || worker.child.is_none() { + if !terminal(state.status) { + let Some(lease) = lease else { + continue; + }; + let outcome = if state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else { + RunOutcome::Failed + }; + publish( + store, + lease, + &worker.claim, + ( + "local-exit", + RunChange::Finished { + outcome, + reason: "worker stopped; physical exit or absence confirmed".into(), + exit_code: worker.exit.and_then(WorkerExit::code), + }, + ), + ) + .await?; + } + self.active.swap_remove(index); + } + } + let Some(lease) = lease else { + return Ok(()); + }; + self.launch_available(store, lease).await + } + + /// Never release claims until children have physically stopped, even if shutdown loses its fence. + async fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + self.shutdown_workers(store, lease).await + } +} + +impl Workers { + async fn launch_available( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + let mut renewed = Instant::now(); + let eligible = self.configuration.dispatch_scopes(); + for location in self.configuration.locations() { + let scope = location.scope(); + let running = self + .active + .iter() + .filter(|worker| { + worker.claim.request.location_id == scope.location_id + && worker.claim.request.target == scope.target + }) + .count(); + for _ in running..scope.max_workers.get() { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } + let Some(claim) = claim_next_run( + store, + lease, + &self.configuration.concurrency_limits, + RunDispatch { + scope: Some((scope.location_id, scope.target)), + eligible: Some(&eligible), + }, + ) + .await? + else { + break; + }; + let index = self.active.len(); + self.active.push(ClaimedWorker { + claim, + child: None, + stopping: None, + exit: None, + launch_failure: None, + }); + let worker = &mut self.active[index]; + let claim = &worker.claim; + if matches!(location, Launcher::Local(_)) && claim.request.image_id.is_some() { + worker.launch_failure = Some("local_image_rejected"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; + publish(store, lease, claim, ("local-image-rejected", RunChange::Finished { outcome: RunOutcome::Failed, reason: "image-pinned runs require a Docker launcher, not a local interpreter".into(), exit_code: None })).await?; + self.active.swap_remove(index); + continue; + } + let container_id = if let Launcher::Docker(location) = location { + match DockerWorker::prepare(location, claim) { + Ok(child) => { + let name = child.name.clone(); + worker.child = Some(WorkerChild::Docker(child)); + Some(name) + } + Err(_) => { + worker.launch_failure = Some("docker_configuration_rejected"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; + publish(store, lease, claim, ("docker-image-rejected", RunChange::Finished { + outcome: RunOutcome::Failed, reason: "Docker worker requires a pinned image and supported launcher configuration".into(), exit_code: None, + })).await?; + self.active.swap_remove(index); + continue; + } + } + } else { + None + }; + let context = AuditContext { + actor: format!("scheduler/{}", lease.holder), + granting_role: "scheduler".into(), + request_id: claim.claim_id, + ip_address: None, + }; + let credential = issue_token( + store, + &context, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + Utc::now() + TimeDelta::days(7), + ) + .await?; + let local_command = match location { + Launcher::Local(location) => match prepare_spawn( + location, + claim, + &credential.bearer, + &self.control_variable, + ) { + Ok(command) => Some(command), + Err(error) => { + worker.launch_failure = Some("local_preparation_failed"); + worker.publish_launch_failure(store, lease).await?; + return Err(error); + } + }, + Launcher::Docker(_) => None, + }; + publish( + store, + lease, + claim, + ("local-launched", RunChange::Launched { container_id }), + ) + .await?; + let launched = match local_command { + Some(mut command) => command + .spawn() + .map(|child| { + worker.child = Some(WorkerChild::Local(child)); + }) + .map_err(ControlError::from), + None => { + if let (Launcher::Docker(location), Some(WorkerChild::Docker(child))) = + (location, &worker.child) + && let Err(error) = child + .launch(location, claim, &credential.bearer, &self.control_variable) + .await + { + worker.launch_failure = Some("docker_launch_failed"); + worker.publish_launch_failure(store, lease).await?; + return Err(error); + } + Ok(()) + } + }; + match launched { + Ok(()) => { + drop(writeln!( + std::io::stderr().lock(), + "Worker launched for run {}", + claim.request.run_id + )); + } + Err(_) => { + worker.launch_failure = Some("local_spawn_failed"); + worker.publish_launch_failure(store, lease).await?; + let claim = &worker.claim; + publish( + store, + lease, + claim, + ( + "local-spawn-failed", + RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "local worker process could not start".into(), + exit_code: None, + }, + ), + ) + .await?; + self.active.swap_remove(index); + } + } + } + } + Ok(()) + } +} + +fn prepare_spawn( + location: &LocalLauncher, + claim: &RunClaim, + bearer: &str, + control_variable: &str, +) -> Result { + let mut command = Command::new(&location.executable); + command + .args(arguments(&Launcher::Local(location).scope(), claim)) + .arg(&location.project_dir) + .current_dir(&location.project_dir) + .env_clear() + .envs(std::env::vars_os().filter(|(key, _)| { + key.to_str() + .is_none_or(|key| !excluded(key, control_variable)) + })) + .envs(environment(claim, bearer)) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .kill_on_drop(true); + #[cfg(target_os = "linux")] + crate::workers::_helpers::local::inherit_claim(&mut command, location, claim)?; + Ok(command) +} + +impl ClaimedWorker { + pub(in crate::workers) async fn publish_launch_failure( + &mut self, + store: &ControlStore, + lease: &SchedulerLease, + ) -> Result<(), ControlError> { + if let Some(code) = self.launch_failure { + publish( + store, + lease, + &self.claim, + (code, RunChange::LaunchFailed { code: code.into() }), + ) + .await?; + self.launch_failure = None; + } + Ok(()) + } +} + +pub(in crate::workers) async fn publish( + store: &ControlStore, + lease: &SchedulerLease, + claim: &RunClaim, + (event, change): (&str, RunChange), +) -> Result<(), ControlError> { + change_run( + store, + RunAuthority::Scheduler(lease), + &RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, event.as_bytes()), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await?; + Ok(()) +} + +pub(in crate::workers) fn terminal(status: RunStatus) -> bool { + !matches!( + status, + RunStatus::Queued | RunStatus::Launched | RunStatus::Running | RunStatus::Unresponsive + ) +} + +pub(in crate::workers) fn exit_code(exit: ExitStatus) -> i64 { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + i64::from( + exit.code() + .unwrap_or_else(|| 128 + exit.signal().unwrap_or(0)), + ) + } + #[cfg(not(unix))] + i64::from(exit.code().unwrap_or(1)) +} + +pub(in crate::workers) fn terminate(child: &mut Child) -> Result<(), ControlError> { + #[cfg(unix)] + if let Some(id) = child.id() { + let raw = i32::try_from(id) + .map_err(|error| ControlError::Invalid(format!("invalid child process ID: {error}")))?; + let pid = rustix::process::Pid::from_raw(raw) + .ok_or_else(|| ControlError::Invalid("invalid child process ID".into()))?; + match rustix::process::kill_process(pid, rustix::process::Signal::TERM) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(std::io::Error::from(error).into()), + } + } + #[cfg(not(unix))] + let _ = child; + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/workers/constants.rs b/crates/sqlbuild-cli/src/workers/constants.rs new file mode 100644 index 0000000000..06a889dbfb --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/constants.rs @@ -0,0 +1,7 @@ +//! Immutable Docker image identities use the full SHA-256 hexadecimal digest. + +pub(super) const IMAGE_ID_HEX_LENGTH: usize = 64; + +/// Keep inherited claim ownership away from standard IO and the runtime's usual low descriptors. +#[cfg(target_os = "linux")] +pub(super) const LOCAL_CLAIM_FD: i32 = 198; diff --git a/crates/sqlbuild-cli/src/workers/main/mod.rs b/crates/sqlbuild-cli/src/workers/main/mod.rs new file mode 100644 index 0000000000..2c35ad2bd4 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod open_workers; diff --git a/crates/sqlbuild-cli/src/workers/main/open_workers.rs b/crates/sqlbuild-cli/src/workers/main/open_workers.rs new file mode 100644 index 0000000000..544dae4fc5 --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/main/open_workers.rs @@ -0,0 +1,19 @@ +//! Open the installation's process owner before accepting scheduler work. + +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::workers::_helpers::configuration::read_configuration; +use crate::workers::models::Workers; + +pub(crate) fn open_workers( + configuration_path: Option<&str>, + control_variable: String, + scheduler_enabled: bool, +) -> Result { + Ok(Workers { + configuration: read_configuration(configuration_path, scheduler_enabled)?, + active: Vec::new(), + control_variable, + recovered_epoch: None, + }) +} diff --git a/crates/sqlbuild-cli/src/workers/mod.rs b/crates/sqlbuild-cli/src/workers/mod.rs new file mode 100644 index 0000000000..59bf71d77a --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/mod.rs @@ -0,0 +1,6 @@ +//! Native process owners launch claim-scoped workers without transferring control-store access. + +mod _helpers; +mod constants; +pub(crate) mod main; +pub(crate) mod models; diff --git a/crates/sqlbuild-cli/src/workers/models.rs b/crates/sqlbuild-cli/src/workers/models.rs new file mode 100644 index 0000000000..35639fe18e --- /dev/null +++ b/crates/sqlbuild-cli/src/workers/models.rs @@ -0,0 +1,111 @@ +//! Installation launch configuration and the physical children retained until reaping. + +use std::collections::BTreeMap; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::path::PathBuf; +use std::time::Instant; + +use sqlbuild_orchestration::runs::models::RunClaim; +use tokio::process::Child; +use uuid::Uuid; + +/// Physical ownership and stop evidence are independent of the scheduler's durable run state. +#[derive(Debug)] +pub(super) enum WorkerChild { + Local(Child), + #[cfg(target_os = "linux")] + RecoveredLocal(LocalWorker), + Docker(DockerWorker), +} + +/// The inherited lock proves physical ownership without relying on a remembered process ID. +#[cfg(target_os = "linux")] +#[derive(Debug)] +pub(super) struct LocalWorker { + pub(super) lock: std::fs::File, + pub(super) run_id: Uuid, + pub(super) stopped: bool, +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum WorkerExit { + Code(i64), + Absent, +} + +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Launchers { + #[serde(default)] + pub(super) local: Vec, + #[serde(default)] + pub(super) docker: Vec, + #[serde(default)] + pub(super) concurrency_limits: BTreeMap, +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct LauncherScope<'a> { + pub(super) location_id: Uuid, + pub(super) target: &'a str, + pub(super) server_url: &'a str, + pub(super) max_workers: NonZeroUsize, +} + +/// No executable, project path or API address is supplied by an untrusted run request. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct LocalLauncher { + pub(super) location_id: Uuid, + pub(super) target: String, + pub(super) server_url: String, + pub(super) max_workers: NonZeroUsize, + pub(super) project_dir: PathBuf, + pub(super) executable: PathBuf, +} + +/// The installation chooses the daemon and location container; the snapshot pins the worker image. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct DockerLauncher { + pub(super) location_id: Uuid, + pub(super) target: String, + pub(super) server_url: String, + pub(super) max_workers: NonZeroUsize, + pub(super) socket: PathBuf, + pub(super) location_container: String, + pub(super) executable: String, +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum Launcher<'a> { + Local(&'a LocalLauncher), + Docker(&'a DockerLauncher), +} + +/// Retain the deterministic identity before create/start, including uncertain HTTP responses. +#[derive(Debug)] +pub(super) struct DockerWorker { + pub(super) client: reqwest::Client, + pub(super) name: String, + pub(super) claim_id: Uuid, + pub(super) image_id: String, +} + +#[derive(Debug)] +pub(super) struct ClaimedWorker { + pub(super) claim: RunClaim, + pub(super) child: Option, + pub(super) stopping: Option, + pub(super) exit: Option, + pub(super) launch_failure: Option<&'static str>, +} + +/// Physical ownership survives leadership loss; only the current fence changes durable lifecycle. +#[derive(Debug)] +pub(crate) struct Workers { + pub(super) configuration: Launchers, + pub(super) active: Vec, + pub(super) control_variable: String, + pub(super) recovered_epoch: Option, +} diff --git a/crates/sqlbuild-cli/tests/native_commands.rs b/crates/sqlbuild-cli/tests/native_commands.rs new file mode 100644 index 0000000000..208f02150b --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands.rs @@ -0,0 +1,12 @@ +#[path = "native_commands/admin/helpers.rs"] +mod admin_helpers; +#[path = "native_commands/helpers.rs"] +mod helpers; +#[path = "native_commands/admin/test_admin.rs"] +mod test_admin; +#[path = "native_commands/test_control.rs"] +mod test_control; +#[path = "native_commands/test_types.rs"] +mod test_types; +#[path = "native_commands/worker/test_worker.rs"] +mod worker; diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs new file mode 100644 index 0000000000..d43fbfa744 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/helpers.rs @@ -0,0 +1,93 @@ +//! Run the isolated native executable; never include returned bearer bytes in assertion messages. + +use std::process::{Command, Output}; + +use serde_json::Value; +use sqlbuild_orchestration::declarations::models::TriggerRecord; +use uuid::Uuid; + +use crate::test_types::TestControl; + +pub(crate) fn administer(control: &TestControl, arguments: &[&str]) -> Value { + let output = invoke_admin(control, arguments); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration completed successfully\n" + ); + serde_json::from_slice(&output.stdout).expect("JSON-only admin stdout") +} + +pub(crate) fn invoke_admin(control: &TestControl, arguments: &[&str]) -> Output { + Command::new( + control + .directory + .path() + .join(format!("sqb{}", std::env::consts::EXE_SUFFIX)), + ) + .arg("admin") + .args(arguments) + .current_dir(control.directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .output() + .expect("native administration process") +} + +pub(crate) async fn read_triggers( + url: &str, + bearer: &str, + (location, target): (Uuid, &str), +) -> Vec { + let mut endpoint = reqwest::Url::parse(url).expect("API address"); + endpoint + .path_segments_mut() + .expect("API path") + .pop_if_empty() + .extend([ + "api", + "locations", + &location.to_string(), + "targets", + target, + "triggers", + ]); + reqwest::Client::new() + .get(endpoint) + .bearer_auth(bearer) + .timeout(std::time::Duration::from_secs(5)) + .send() + .await + .expect("trigger response") + .error_for_status() + .expect("authorized trigger read") + .json() + .await + .expect("trigger records") +} + +pub(crate) fn audit_actions(control: &TestControl, bearer: &str) -> Vec { + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("inspect control audit"); + let mut query = connection.prepare("SELECT action, actor, granting_role, scope_json || before_json || after_json FROM sqb_security_audit ORDER BY occurred_at, event_id") + .expect("audit records"); + query + .query_map([], |row| { + let action: String = row.get(0)?; + let actor: String = row.get(1)?; + let role: String = row.get(2)?; + let payload: String = row.get(3)?; + assert_eq!(actor, "local-control-store-owner"); + assert_eq!(role, "control-store-owner"); + assert!(!payload.contains(bearer)); + Ok(action) + }) + .expect("read audit") + .collect::>() + .expect("decode audit") +} diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs new file mode 100644 index 0000000000..f2ab692398 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_admin.rs @@ -0,0 +1,344 @@ +//! Native bootstrap credentials authorize real publication and stop working after revocation. + +#[path = "test_types.rs"] +mod test_types; + +use std::collections::BTreeMap; + +use chrono::{Duration, Utc}; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use uuid::Uuid; + +use crate::admin_helpers::{administer, audit_actions, invoke_admin, read_triggers}; +use crate::helpers::{api_ready, start_control}; +#[cfg(unix)] +use crate::test_admin::test_types::AdminDeliveryTestCase; +use crate::test_admin::test_types::{AdminTestCase, PrincipalAdminTestCase, TriggerAdminTestCase}; + +#[test] +fn given_a_standalone_control_server_when_bootstrapped_by_an_operator_then_scoped_publication_and_revocation_are_audited() + { + let test_cases = [AdminTestCase { + description: "native operator bootstrap preserves labels and scopes credentials", + location_name: "orders / café", + expected_name: "orders / café", + expected_actions: [ + "location.created", + "location.created", + "token.issued", + "token.revoked", + "trigger.changed", + "trigger.changed", + "trigger.changed", + "trigger.changed", + "trigger.changed", + ], + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let first = administer(&control, &["create-location", test_case.location_name]); + let second = administer(&control, &["create-location", "customers"]); + let location_id = + Uuid::parse_str(first["location_id"].as_str().expect("location UUID")) + .expect("UUID"); + let other_id = Uuid::parse_str(second["location_id"].as_str().expect("location UUID")) + .expect("UUID"); + assert_ne!(location_id, other_id); + assert_eq!( + first["name"], test_case.expected_name, + "{}", + test_case.description + ); + let expiry = (Utc::now() + Duration::hours(1)).to_rfc3339(); + let credential = administer( + &control, + &[ + "issue-location-token", + &location_id.to_string(), + "--expires-at", + &expiry, + ], + ); + let bearer = credential["bearer"].as_str().expect("one-time bearer"); + assert_eq!(credential["identity"]["scope"]["kind"], "location"); + assert_eq!( + credential["identity"]["scope"]["location_id"], + location_id.to_string() + ); + let mut snapshot = SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([("model:orders".into(), SnapshotNode { + kind: "model".into(), + definition: serde_json::json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + })]), + documents: BTreeMap::from([ + ("jobs".into(), serde_json::json!({"orders": {"description": "Refresh orders.", "selection": ["model:orders"]}})), + ("triggers".into(), serde_json::json!({"daily_orders": { + "description": "Daily orders.", "job_key": "orders", "default_status": "paused", + "schedule": {"kind": "cron", "expression": "0 2 * * *", "timezone": "UTC"}, + }})), + ]), + }; + let published = publish_remote_snapshot(&url, bearer, &snapshot) + .await + .expect("operator-issued credential publishes"); + assert_eq!(published.location_id, location_id); + let trigger_id = read_triggers(&url, bearer, (location_id, "development")).await[0].trigger_id.to_string(); + for case in [ + TriggerAdminTestCase { description: "enable overrides the paused code default", change: "enable", cursor_arguments: &[], expected_status: "running", expected_cursor: None }, + TriggerAdminTestCase { description: "pause overrides an enabled trigger", change: "pause", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + TriggerAdminTestCase { description: "reset-default restores the paused code default", change: "reset-default", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + TriggerAdminTestCase { description: "set-cursor normalizes the timestamp offset", change: "set-cursor", cursor_arguments: &["--cursor", "2026-07-03T01:17:23+05:30"], expected_status: "paused", expected_cursor: Some("2026-07-02T19:47:23+00:00") }, + TriggerAdminTestCase { description: "reset-cursor removes the durable cursor", change: "reset-cursor", cursor_arguments: &[], expected_status: "paused", expected_cursor: None }, + ] { + let event_id = Uuid::now_v7().to_string(); + let mut arguments = vec!["change-trigger", &trigger_id, case.change, "--event-id", &event_id, "--reason", "Order refresh maintenance"]; + arguments.extend_from_slice(case.cursor_arguments); + for expected in [true, false] { + let result = administer(&control, &arguments); + assert_eq!(result["applied"], expected, "{}", case.description); + assert_eq!(result["event_id"], event_id); + } + let mut conflict = arguments.clone(); + conflict[6] = "Different request content"; + assert!(!invoke_admin(&control, &conflict).status.success(), "{}", case.description); + let triggers = read_triggers(&url, bearer, (location_id, "development")).await; + assert_eq!(triggers[0].status.as_str(), case.expected_status, "{}", case.description); + assert_eq!(triggers[0].last_tick_for.map(|value| value.to_rfc3339()).as_deref(), case.expected_cursor, "{}", case.description); + } + snapshot.generation = 2; + snapshot.documents.get_mut("triggers").expect("trigger document")["daily_orders"]["default_status"] = serde_json::json!("running"); + publish_remote_snapshot(&url, bearer, &snapshot).await.expect("publish a new default"); + assert_eq!(read_triggers(&url, bearer, (location_id, "development")).await[0].status.as_str(), "running", "reset-default removed the operator override"); + snapshot.location_id = other_id; + assert!(matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::FORBIDDEN)) + )); + snapshot.location_id = location_id; + snapshot.generation = 2; + let token_id = credential["identity"]["token_id"] + .as_str() + .expect("credential ID"); + for expected in [true, false] { + let revoked = administer(&control, &["revoke-token", token_id]); + assert_eq!(revoked["revoked"], expected); + assert_eq!(revoked["token_id"], token_id); + } + assert!(matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); + let mut actions = audit_actions(&control, bearer); + actions.sort(); + assert_eq!( + actions, test_case.expected_actions, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_native_principal_administration_when_issuing_credentials_then_identity_is_audited_without_granting_location_permissions() + { + let test_cases = [ + PrincipalAdminTestCase { + description: "a person credential has no implicit location authority", + kind: "person", + expected_name: "orders operator / café", + expected_actions: ["principal.created", "token.issued", "token.revoked"], + }, + PrincipalAdminTestCase { + description: "a service account credential has no implicit location authority", + kind: "service_account", + expected_name: "orders automation", + expected_actions: ["principal.created", "token.issued", "token.revoked"], + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let principal = administer( + &control, + &[ + "create-principal", + test_case.expected_name, + "--kind", + test_case.kind, + ], + ); + let principal_id = Uuid::parse_str( + principal["principal_id"] + .as_str() + .expect(test_case.description), + ) + .expect(test_case.description); + assert_eq!( + principal["name"], test_case.expected_name, + "{}", + test_case.description + ); + assert_eq!( + principal["kind"], test_case.kind, + "{}", + test_case.description + ); + let expiry = (Utc::now() + Duration::hours(1)).to_rfc3339(); + let credential = administer( + &control, + &[ + "issue-principal-token", + &principal_id.to_string(), + "--expires-at", + &expiry, + ], + ); + assert_eq!( + credential["identity"]["scope"]["kind"], "principal", + "{}", + test_case.description + ); + assert_eq!( + credential["identity"]["scope"]["principal_id"], + principal_id.to_string(), + "{}", + test_case.description + ); + let bearer = credential["bearer"].as_str().expect(test_case.description); + let snapshot = SnapshotInput { + location_id: Uuid::now_v7(), + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::new(), + documents: BTreeMap::new(), + }; + assert!( + matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::FORBIDDEN)) + ), + "{}", + test_case.description + ); + let token_id = credential["identity"]["token_id"] + .as_str() + .expect(test_case.description); + let revoked = administer(&control, &["revoke-token", token_id]); + assert_eq!(revoked["revoked"], true, "{}", test_case.description); + assert!( + matches!( + publish_remote_snapshot(&url, bearer, &snapshot).await, + Err(ClientError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + ), + "{}", + test_case.description + ); + assert_eq!( + audit_actions(&control, bearer), + test_case.expected_actions, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[cfg(unix)] +#[test] +fn given_a_committed_admin_operation_when_json_output_is_lost_then_all_response_sizes_fail_without_claiming_rollback() + { + let test_cases = [ + AdminDeliveryTestCase { + description: "small response", + name_bytes: 8, + expected_exit: 1, + expected_rows: (1, 1), + }, + AdminDeliveryTestCase { + description: "Python compatibility drop window", + name_bytes: 5000, + expected_exit: 1, + expected_rows: (1, 1), + }, + AdminDeliveryTestCase { + description: "large response", + name_bytes: 9000, + expected_exit: 1, + expected_rows: (1, 1), + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated administration directory"); + let binary = crate::helpers::isolated_binary(directory.path()); + let output = std::process::Command::new(binary) + .args([ + "admin", + "create-location", + &"x".repeat(test_case.name_bytes), + ]) + .current_dir(directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .stdout( + std::fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .expect("failing output device"), + ) + .output() + .expect("native administration"); + assert_eq!( + output.status.code(), + Some(test_case.expected_exit), + "{}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Opening control store...\nControl store ready\nControl administration JSON result delivery failed; changes may already be committed\n", + "{}", + test_case.description + ); + let connection = + rusqlite::Connection::open(directory.path().join(".sqlbuild/control.sqlite3")) + .expect("inspect committed operation"); + let rows: (i64, i64) = connection.query_row("SELECT (SELECT COUNT(*) FROM sqb_locations), (SELECT COUNT(*) FROM sqb_security_audit)", [], |row| Ok((row.get(0)?, row.get(1)?))).expect("location and audit remain committed"); + assert_eq!(rows, test_case.expected_rows, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs new file mode 100644 index 0000000000..3adc9b5e24 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/admin/test_types.rs @@ -0,0 +1,29 @@ +pub(crate) struct AdminTestCase { + pub(crate) description: &'static str, + pub(crate) location_name: &'static str, + pub(crate) expected_name: &'static str, + pub(crate) expected_actions: [&'static str; 9], +} + +pub(crate) struct PrincipalAdminTestCase { + pub(crate) description: &'static str, + pub(crate) kind: &'static str, + pub(crate) expected_name: &'static str, + pub(crate) expected_actions: [&'static str; 3], +} + +pub(crate) struct TriggerAdminTestCase { + pub(crate) description: &'static str, + pub(crate) change: &'static str, + pub(crate) cursor_arguments: &'static [&'static str], + pub(crate) expected_status: &'static str, + pub(crate) expected_cursor: Option<&'static str>, +} + +#[cfg(unix)] +pub(crate) struct AdminDeliveryTestCase { + pub(crate) description: &'static str, + pub(crate) name_bytes: usize, + pub(crate) expected_exit: i32, + pub(crate) expected_rows: (i64, i64), +} diff --git a/crates/sqlbuild-cli/tests/native_commands/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/helpers.rs new file mode 100644 index 0000000000..ec2421c381 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/helpers.rs @@ -0,0 +1,144 @@ +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::test_types::{ServiceTestCase, TestControl}; + +pub(crate) fn isolated_binary(directory: &Path) -> std::path::PathBuf { + let binary = directory.join(format!("sqb{}", std::env::consts::EXE_SUFFIX)); + // Do not open the executable for writing: concurrent process launches can inherit + // the copy's writable descriptor until exec, making this launch fail with ETXTBSY. + std::fs::hard_link(env!("CARGO_BIN_EXE_sqb"), &binary) + .expect("link standalone binary without Python CLI"); + binary +} + +pub(crate) fn start_control(arguments: &[&str]) -> TestControl { + let directory = tempfile::tempdir().expect("isolated native control project"); + let binary = isolated_binary(directory.path()); + let stderr = directory.path().join("control.log"); + let child = Command::new(binary) + .args(arguments) + .current_dir(directory.path()) + .env("PATH", "") + .env_remove("SQLBUILD_CONTROL_DATABASE_URL") + .stdout(Stdio::null()) + .stderr(std::fs::File::create(&stderr).expect("control stderr")) + .spawn() + .expect("start native control process"); + TestControl { + child, + directory, + stderr, + } +} + +pub(crate) async fn api_ready(control: &mut TestControl) -> String { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .expect("HTTP client"); + tokio::time::timeout(Duration::from_secs(10), async { + for _ in 0..500 { + assert!( + control.child.try_wait().expect("child status").is_none(), + "control process exited: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + let log = std::fs::read_to_string(&control.stderr).expect("read control log"); + let Some(address) = log + .lines() + .find_map(|line| line.strip_prefix("Control API ready: ")) + else { + continue; + }; + let url = format!("http://{address}"); + let Ok(response) = client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .and_then(reqwest::Response::error_for_status) + else { + continue; + }; + drop(response); + return url; + } + panic!("native API exhausted readiness attempts"); + }) + .await + .expect("native API becomes ready") +} + +pub(crate) fn health_counts(control: &TestControl) -> Option<(i64, i64)> { + let connection = rusqlite::Connection::open_with_flags( + control.directory.path().join(".sqlbuild/control.sqlite3"), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .ok()?; + connection + .query_row( + "SELECT (SELECT COUNT(*) FROM sqb_scheduler_lease), \ + (SELECT COUNT(*) FROM recall_events WHERE event_type='platform_heartbeat')", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .ok() +} + +pub(crate) async fn wait_for_health( + control: &mut TestControl, + case: &ServiceTestCase, +) -> (i64, i64) { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + assert!( + control.child.try_wait().expect("service status").is_none(), + "service exited: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + let Some(counts) = health_counts(control).filter(|(leaders, heartbeats)| { + *leaders == case.expected_leaders && *heartbeats >= case.expected_heartbeats + }) else { + continue; + }; + return counts; + } + panic!("control service exhausted health readiness attempts"); + }) + .await + .expect("native control service publishes its health state") +} + +#[cfg(unix)] +pub(crate) async fn terminate(control: &mut TestControl) { + let status = Command::new("kill") + .args(["-TERM", &control.child.id().to_string()]) + .status() + .expect("send SIGTERM"); + assert!(status.success()); + let status = tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + tokio::time::sleep(Duration::from_millis(20)).await; + let Some(status) = control.child.try_wait().expect("child exit") else { + continue; + }; + return status; + } + panic!("control roles exhausted shutdown attempts"); + }) + .await + .expect("control roles stop promptly"); + assert!( + status.success(), + "shutdown failed: {}", + std::fs::read_to_string(&control.stderr).unwrap_or_default() + ); + assert!( + std::fs::read_to_string(&control.stderr) + .expect("shutdown log") + .ends_with("Control service stopped successfully\n") + ); +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs new file mode 100644 index 0000000000..215540fdb2 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -0,0 +1,441 @@ +use std::process::Command; + +use crate::helpers::{api_ready, isolated_binary, start_control, wait_for_health}; +use crate::test_types::{CommandTestCase, OpenApiTestCase, ServiceTestCase, TriggerCursorTestCase}; + +#[test] +fn given_a_standalone_binary_when_printing_help_or_rejecting_invalid_input_then_python_and_database_access_are_not_required() + { + let test_cases = [ + CommandTestCase { + description: "native server help", + arguments: &["server", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb ", + expected_stderr: "", + }, + CommandTestCase { + description: "native scheduler help", + arguments: &["scheduler", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb ", + expected_stderr: "", + }, + CommandTestCase { + description: "native privileged administration help", + arguments: &["admin", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin ", + expected_stderr: "", + }, + CommandTestCase { + description: "native trigger administration help", + arguments: &["admin", "change-trigger", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin change-trigger ", + expected_stderr: "", + }, + CommandTestCase { + description: "native cancellation help", + arguments: &["admin", "cancel-run", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin cancel-run ", + expected_stderr: "", + }, + CommandTestCase { + description: "native manual job request help", + arguments: &["admin", "request-run", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin request-run ", + expected_stderr: "", + }, + CommandTestCase { + description: "native location work help", + arguments: &["admin", "check-location", "--help"], + expected_success: true, + expected_stdout_prefix: "usage: sqb admin check-location ", + expected_stderr: "", + }, + CommandTestCase { + description: "blank location work reason rejected before store initialization", + arguments: &[ + "admin", + "check-location", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + "--request-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: target and reason must not be blank\n", + }, + CommandTestCase { + description: "blank job rejected before store initialization", + arguments: &[ + "admin", + "request-run", + "00000000-0000-0000-0000-000000000001", + " ", + "--target", + "development", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: target and job must not be blank\n", + }, + CommandTestCase { + description: "oversized manual queue priority rejected before store initialization", + arguments: &[ + "admin", + "request-run", + "00000000-0000-0000-0000-000000000001", + "daily_orders", + "--target", + "development", + "--run-id", + "00000000-0000-0000-0000-000000000002", + "--priority", + "9223372036854775808", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --priority must fit a signed 64-bit integer\n", + }, + CommandTestCase { + description: "blank cancellation reason rejected before store initialization", + arguments: &[ + "admin", + "cancel-run", + "00000000-0000-0000-0000-000000000001", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --reason must not be blank\n", + }, + CommandTestCase { + description: "blank trigger reason rejected before store initialization", + arguments: &[ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + "pause", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + " ", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --reason must not be blank\n", + }, + CommandTestCase { + description: "set-cursor requires a timestamp before store initialization", + arguments: &[ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + "set-cursor", + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: trigger change: missing field `time`\n", + }, + CommandTestCase { + description: "blank location rejected before store initialization", + arguments: &["admin", "create-location", " "], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: location name is required\n", + }, + CommandTestCase { + description: "blank principal rejected before store initialization", + arguments: &["admin", "create-principal", " "], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: principal name is required\n", + }, + CommandTestCase { + description: "invalid credential ID rejected before store initialization", + arguments: &["admin", "revoke-token", "not-a-uuid"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: a valid subject UUID is required\n", + }, + CommandTestCase { + description: "invalid expiry rejected before store initialization", + arguments: &[ + "admin", + "issue-location-token", + "00000000-0000-0000-0000-000000000001", + "--expires-at", + "tomorrow", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: --expires-at must be an RFC3339 timestamp\n", + }, + CommandTestCase { + description: "expired credential rejected before store initialization", + arguments: &[ + "admin", + "issue-location-token", + "00000000-0000-0000-0000-000000000001", + "--expires-at", + "2000-01-01T00:00:00Z", + ], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control administration failed: invalid control-plane data: token expiry must be in the future\n", + }, + CommandTestCase { + description: "invalid port is rejected before store initialization", + arguments: &["server", "--port", "65536"], + expected_success: false, + expected_stdout_prefix: "", + expected_stderr: "Control service failed: invalid control-plane data: --port must be between 0 and 65535\n", + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + let output = Command::new(&binary) + .args(test_case.arguments) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone native command"); + assert_eq!( + output.status.success(), + test_case.expected_success, + "{}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + test_case.expected_stderr, + "{}", + test_case.description + ); + assert!( + String::from_utf8_lossy(&output.stdout).starts_with(test_case.expected_stdout_prefix), + "{}", + test_case.description + ); + assert!( + !directory.path().join(".sqlbuild").exists(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_cursor_free_trigger_change_when_a_cursor_is_supplied_then_it_fails_before_opening_the_store() + { + let test_cases = [ + TriggerCursorTestCase { + description: "enable rejects an extra cursor", + change: "enable", + expected_success: false, + }, + TriggerCursorTestCase { + description: "pause rejects an extra cursor", + change: "pause", + expected_success: false, + }, + TriggerCursorTestCase { + description: "reset-default rejects an extra cursor", + change: "reset-default", + expected_success: false, + }, + TriggerCursorTestCase { + description: "reset-cursor rejects an extra cursor", + change: "reset-cursor", + expected_success: false, + }, + ]; + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let binary = isolated_binary(directory.path()); + for test_case in test_cases { + for cursor in ["2026-01-01T00:00:00Z", "nonsense"] { + let output = Command::new(&binary) + .args([ + "admin", + "change-trigger", + "00000000-0000-0000-0000-000000000001", + test_case.change, + "--event-id", + "00000000-0000-0000-0000-000000000002", + "--reason", + "Refresh orders", + "--cursor", + cursor, + ]) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone native command"); + assert_eq!( + output.status.success(), + test_case.expected_success, + "{}: {cursor}", + test_case.description + ); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "Control administration failed: invalid control-plane data: --cursor is only valid for set-cursor\n", + "{}: {cursor}", + test_case.description + ); + assert!( + output.stdout.is_empty(), + "{}: {cursor}", + test_case.description + ); + assert!(!directory.path().join(".sqlbuild").exists()); + } + } +} + +#[test] +fn given_a_standalone_server_when_exporting_openapi_then_the_worker_contract_is_machine_readable_without_python_or_a_database() + { + let test_cases = [OpenApiTestCase { + description: "worker lifecycle endpoint", + expected_path: "/api/runs/{run_id}/events", + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("isolated CLI directory"); + let output = Command::new(isolated_binary(directory.path())) + .args(["server", "--openapi"]) + .current_dir(directory.path()) + .env("PATH", "") + .env("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://not-opened") + .output() + .expect("standalone OpenAPI"); + assert!(output.status.success(), "{}", test_case.description); + assert!(output.stderr.is_empty(), "{}", test_case.description); + let document: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("machine-readable OpenAPI"); + assert!( + document["paths"][test_case.expected_path].is_object(), + "{}", + test_case.description + ); + assert!( + !directory.path().join(".sqlbuild").exists(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_standalone_binary_when_serving_with_or_without_a_scheduler_then_migrations_auth_and_shutdown_are_native() + { + let test_cases = [ + ServiceTestCase { + description: "server without scheduling still observes platform health", + arguments: &["server", "--port", "0", "--no-scheduler"], + expected_leaders: 0, + expected_heartbeats: 1, + }, + ServiceTestCase { + description: "server with embedded scheduling", + arguments: &["server", "--port", "0"], + expected_leaders: 1, + expected_heartbeats: 1, + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(test_case.arguments); + let url = api_ready(&mut control).await; + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .build() + .expect("bounded HTTP requests"); + let response = client + .post(format!("{url}/api/runs/{}/heartbeat", uuid::Uuid::now_v7())) + .send() + .await + .expect("protected native API"); + assert_eq!( + response.status(), + reqwest::StatusCode::UNAUTHORIZED, + "{}", + test_case.description + ); + let (leaders, heartbeats) = wait_for_health(&mut control, &test_case).await; + assert_eq!( + leaders, test_case.expected_leaders, + "{}", + test_case.description + ); + assert!( + heartbeats >= test_case.expected_heartbeats, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} + +#[test] +fn given_a_standalone_scheduler_when_started_without_python_then_it_migrates_and_takes_the_durable_lease() + { + let test_cases = [ServiceTestCase { + description: "standalone scheduler takes a lease without starting a server watchdog", + arguments: &["scheduler"], + expected_leaders: 1, + expected_heartbeats: 0, + }]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(test_case.arguments); + let (leaders, heartbeats) = wait_for_health(&mut control, &test_case).await; + assert_eq!( + leaders, test_case.expected_leaders, + "{}", + test_case.description + ); + assert_eq!( + heartbeats, test_case.expected_heartbeats, + "{}", + test_case.description + ); + #[cfg(unix)] + crate::helpers::terminate(&mut control).await; + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/test_types.rs new file mode 100644 index 0000000000..4796bc1277 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/test_types.rs @@ -0,0 +1,41 @@ +use std::path::PathBuf; +use std::process::Child; + +pub(crate) struct CommandTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_success: bool, + pub(crate) expected_stdout_prefix: &'static str, + pub(crate) expected_stderr: &'static str, +} + +pub(crate) struct TriggerCursorTestCase { + pub(crate) description: &'static str, + pub(crate) change: &'static str, + pub(crate) expected_success: bool, +} + +pub(crate) struct OpenApiTestCase { + pub(crate) description: &'static str, + pub(crate) expected_path: &'static str, +} + +pub(crate) struct ServiceTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_leaders: i64, + pub(crate) expected_heartbeats: i64, +} + +pub(crate) struct TestControl { + pub(crate) child: Child, + pub(crate) directory: tempfile::TempDir, + pub(crate) stderr: PathBuf, +} + +impl Drop for TestControl { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs new file mode 100644 index 0000000000..5a5a7fc4a0 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/helpers.rs @@ -0,0 +1,367 @@ +//! Privileged test operators stay separate from the API-only worker session. + +use std::collections::BTreeMap; +use std::process::{Command, ExitStatus, Stdio}; +use std::time::{Duration, SystemTime}; + +use sqlbuild_orchestration::credentials::main::create_location::create_location; +use sqlbuild_orchestration::credentials::main::issue_token::issue_token; +use sqlbuild_orchestration::credentials::main::revoke_token::revoke_token; +use sqlbuild_orchestration::credentials::models::{AuditContext, CredentialScope, IssuedToken}; +use sqlbuild_orchestration::runs::main::change_run::change_run; +use sqlbuild_orchestration::runs::main::claim_next_run::claim_next_run; +use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use sqlbuild_orchestration::scheduler::main::run_scheduler::run_scheduler; +use sqlbuild_orchestration::scheduler::models::{SchedulerLease, SchedulerState}; +use sqlbuild_orchestration::scheduler::types::RunLauncher; +use sqlbuild_orchestration::snapshots::main::publish_snapshot::publish_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::main::open_store::open_store; +use sqlbuild_orchestration::store::models::ControlStore; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_orchestration::worker::models::WorkerSession; +use uuid::Uuid; + +use crate::admin_helpers::administer; +use crate::helpers::isolated_binary; +use crate::test_types::TestControl; +use crate::worker::test_types::{LeaseCapture, WorkerControl}; + +impl RunLauncher for LeaseCapture { + async fn poll( + &mut self, + _store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + if let Some(lease) = lease { + self.lease + .take() + .expect("first scheduler lease") + .send(lease.clone()) + .expect("lease receiver"); + self.shutdown.send(true).expect("scheduler shutdown"); + } + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &ControlStore, + _lease: Option<&SchedulerLease>, + ) -> Result<(), ControlError> { + Ok(()) + } +} + +pub(crate) async fn start_claimed_worker( + control: &TestControl, + url: &str, +) -> (WorkerSession, WorkerControl) { + let (worker, baseline, token) = claim_worker_run(control, None).await; + change_run( + &worker.store, + RunAuthority::Scheduler(&worker.lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: worker.identity.0, + claim_id: Some(worker.identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("launched worker"); + let client = connect_worker(url, &token.bearer, worker.identity).expect("worker connection"); + let session = start_worker(&client, &baseline) + .await + .expect("verified worker"); + (session, worker) +} + +pub(crate) async fn claim_worker_run( + control: &TestControl, + image_id: Option<&str>, +) -> (WorkerControl, SnapshotInput, IssuedToken) { + let path = control.directory.path().join(".sqlbuild/control.sqlite3"); + let store = open_store(path.to_str().expect("test database path")) + .await + .expect("native control store"); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(&store, "orders", &context) + .await + .expect("location"); + let baseline = SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: image_id.map(str::to_owned), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: serde_json::json!({"materialization": "table"}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([( + "jobs".into(), + serde_json::json!({"daily_orders": { + "description": "Build orders", "selection": ["orders"], "overlap": "allow", "settings": {"full_refresh": true} + }}), + )]), + }; + publish_snapshot(&store, &baseline).await.expect("snapshot"); + let run_id = Uuid::now_v7(); + let location = location_id.to_string(); + let run = run_id.to_string(); + let arguments = [ + "request-run", + &location, + "daily_orders", + "--target", + "development", + "--run-id", + &run, + "--priority", + "-3", + "--reason", + "Refresh orders", + ]; + let requested = administer(control, &arguments); + assert_eq!(requested["run_id"], run); + assert_eq!(requested["applied"], true); + assert_eq!(administer(control, &arguments)["applied"], false); + let connection = rusqlite::Connection::open(&path).expect("audit connection"); + let count: i64 = connection.query_row( + "SELECT COUNT(*) FROM sqb_security_audit WHERE action='job.run_requested' AND actor='local-control-store-owner' AND granting_role='control-store-owner' AND json_extract(scope_json,'$.run_id')=?1", + [&run], |row| row.get(0)).expect("trusted manual request audit"); + assert_eq!(count, 1); + drop(connection); + let (lease_sender, lease_receiver) = tokio::sync::oneshot::channel(); + let (shutdown, signal) = tokio::sync::watch::channel(false); + let (_changes, changes) = tokio::sync::watch::channel(()); + tokio::time::timeout( + Duration::from_secs(5), + run_scheduler( + &store, + SchedulerState::new(Uuid::now_v7()), + (signal, changes), + LeaseCapture { + lease: Some(lease_sender), + shutdown, + }, + ), + ) + .await + .expect("scheduler acquires leadership promptly") + .expect("scheduler shutdown"); + let lease = lease_receiver.await.expect("public scheduler leadership"); + let claim = claim_next_run( + &store, + &lease, + &BTreeMap::new(), + Some((location_id, "development")), + ) + .await + .expect("claim") + .expect("queued work"); + assert_eq!(claim.request.priority, -3); + assert_eq!( + claim.request.settings.get("full_refresh"), + Some(&serde_json::json!(true)) + ); + let identity = (run_id, claim.claim_id); + let token = issue_token( + &store, + &context, + CredentialScope::Run { + run_id, + claim_id: identity.1, + }, + (SystemTime::now() + Duration::from_secs(3600)).into(), + ) + .await + .expect("run-scoped token"); + ( + WorkerControl { + store, + context, + lease, + token_id: token.identity.token_id, + identity, + }, + baseline, + token, + ) +} + +pub(crate) fn cancel_worker(worker: WorkerControl, control: &mut TestControl) { + let event_id = Uuid::now_v7().to_string(); + let run_id = worker.identity.0.to_string(); + let arguments = [ + "cancel-run", + &run_id, + "--event-id", + &event_id, + "--reason", + "Stop the orders build", + ]; + let applied = administer(control, &arguments); + assert_eq!(applied["run_id"], run_id); + assert_eq!(applied["event_id"], event_id); + assert_eq!(applied["applied"], true); + assert_eq!(administer(control, &arguments)["applied"], false); + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("observe durable cancellation audit"); + connection + .query_row( + "SELECT actor, granting_role, scope_json, before_json, after_json FROM sqb_security_audit WHERE event_id=?1 AND action='run.cancel_requested'", + [&event_id], + |row| { + let actor: String = row.get(0)?; + let role: String = row.get(1)?; + let scope: String = row.get(2)?; + let before: String = row.get(3)?; + let after: String = row.get(4)?; + let scope: serde_json::Value = serde_json::from_str(&scope).expect("audit scope"); + let before: serde_json::Value = serde_json::from_str(&before).expect("before state"); + let after: serde_json::Value = serde_json::from_str(&after).expect("after state"); + assert_eq!(actor, "local-control-store-owner"); + assert_eq!(role, "control-store-owner"); + assert_eq!(scope["run_id"], run_id); + assert_eq!(scope["target"], "development"); + assert!(scope["location_id"].is_string()); + assert_eq!(before["status"], "running"); + assert!(before["cancel_requested_at"].is_null()); + assert_eq!(after["status"], "running"); + assert!(after["cancel_requested_at"].is_string()); + assert_eq!(after["reason"], "Stop the orders build"); + Ok(()) + }, + ) + .expect("same-transaction operator history"); + let claims: i64 = connection + .query_row( + "SELECT COUNT(*) FROM sqb_write_claims WHERE run_id=?1", + [&run_id], + |row| row.get(0), + ) + .expect("physical claims after cancellation request"); + assert_eq!( + claims, 1, + "cancellation is not proof of physical termination" + ); +} + +pub(crate) fn revoke_worker(worker: WorkerControl, _control: &mut TestControl) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("independent operator runtime"); + assert!( + runtime + .block_on(revoke_token( + &worker.store, + &worker.context, + worker.token_id + )) + .expect("revoke worker credential") + ); +} + +pub(crate) fn disconnect_worker(_worker: WorkerControl, control: &mut TestControl) { + control.child.kill().expect("stop API process"); + control.child.wait().expect("reap API process"); +} + +pub(crate) fn start_recovery_scheduler( + control: &TestControl, + url: &str, + location_id: Uuid, +) -> TestControl { + let directory = tempfile::tempdir().expect("recovery scheduler logs"); + let configuration = directory.path().join("launchers.json"); + let launcher = serde_json::json!({"docker": [{ + "location_id": location_id, "target": "development", "server_url": url, + "max_workers": 1, "socket": directory.path().join("docker.sock"), + "location_container": "orders-location", "executable": "sqb" + }]}); + std::fs::write(&configuration, launcher.to_string()).expect("Docker scope configuration"); + let stderr = directory.path().join("scheduler.log"); + let child = Command::new(isolated_binary(directory.path())) + .args([ + "scheduler", + "--database-env", + "SQLBUILD_RECOVERY_TEST_STORE", + "--launchers", + ]) + .arg(configuration) + .current_dir(directory.path()) + .env("PATH", "") + .env( + "SQLBUILD_RECOVERY_TEST_STORE", + control.directory.path().join(".sqlbuild/control.sqlite3"), + ) + .stdout(Stdio::null()) + .stderr(std::fs::File::create(&stderr).expect("scheduler stderr")) + .spawn() + .expect("replacement scheduler"); + TestControl { + child, + directory, + stderr, + } +} + +pub(crate) async fn wait_for_worker_log(control: &mut TestControl, expected: &str) { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + let log = std::fs::read_to_string(&control.stderr).expect("worker supervision log"); + assert!( + control + .child + .try_wait() + .expect("scheduler status") + .is_none(), + "{log}" + ); + let Some(_) = log.find(expected) else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + return; + } + panic!("worker supervision exhausted readiness attempts"); + }) + .await + .expect("worker supervision phase completes"); +} + +pub(crate) async fn wait_for_worker_exit(control: &mut TestControl) -> ExitStatus { + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + let Some(exit) = control.child.try_wait().expect("scheduler exit") else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + return exit; + } + panic!("worker supervision exhausted shutdown attempts"); + }) + .await + .expect("shutdown publishes retained terminal evidence") +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs new file mode 100644 index 0000000000..011e0e5490 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_heartbeat.rs @@ -0,0 +1,63 @@ +//! The standalone server remains runnable while the worker's runtime is deliberately blocked. + +use std::time::{Duration, Instant}; + +use crate::helpers::{api_ready, start_control}; +use crate::worker::helpers::{ + cancel_worker, disconnect_worker, revoke_worker, start_claimed_worker, +}; +use crate::worker::test_types::WorkerHeartbeatTestCase; + +#[test] +fn given_blocking_worker_work_when_control_changes_then_heartbeats_remain_independent() { + let test_cases = [ + WorkerHeartbeatTestCase { + description: "cancellation reaches a blocked worker runtime", + interruption: cancel_worker, + expected_cancelled: true, + }, + WorkerHeartbeatTestCase { + description: "credential revocation reaches a blocked worker runtime", + interruption: revoke_worker, + expected_cancelled: true, + }, + WorkerHeartbeatTestCase { + description: "temporary API unavailability does not cancel warehouse work", + interruption: disconnect_worker, + expected_cancelled: false, + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("worker runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let (session, worker) = start_claimed_worker(&control, &url).await; + let operator = std::thread::spawn(move || { + std::thread::sleep(Duration::from_secs(2)); + (test_case.interruption)(worker, &mut control); + control + }); + // No Tokio IO can run on this caller until after the assertion. A heartbeat spawned + // on this runtime, or sharing its connection pool, cannot deliver cancellation. + std::thread::sleep(Duration::from_secs(12)); + assert_eq!( + session.cancel_requested(), + test_case.expected_cancelled, + "{}", + test_case.description + ); + let stopping = Instant::now(); + drop(session); + assert!( + stopping.elapsed() < Duration::from_secs(2), + "{}", + test_case.description + ); + drop(operator.join().expect("operator completed")); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs new file mode 100644 index 0000000000..0a723a1db4 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_recovery.rs @@ -0,0 +1,114 @@ +//! A committed but unlaunchable claim must not poison every replacement scheduler. + +use std::time::Duration; + +use crate::helpers::{api_ready, start_control}; +use crate::worker::helpers::{ + claim_worker_run, start_recovery_scheduler, wait_for_worker_exit, wait_for_worker_log, +}; +use crate::worker::test_types::WorkerRecoveryTestCase; + +#[test] +fn given_an_unlaunchable_committed_claim_when_recovering_then_rejection_retries_keep_ownership() { + let test_cases = [ + WorkerRecoveryTestCase { + description: "snapshot without an image pin", + image_id: None, + failure_event: "run_terminated", + expected_pending_observations: 1, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "snapshot with a malformed image pin", + image_id: Some("sha256:invalid"), + failure_event: "run_terminated", + expected_pending_observations: 1, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "missing image pin retains a rolled-back launch observation", + image_id: None, + failure_event: "platform_launch_failed", + expected_pending_observations: 0, + expected_status: "failed", + }, + WorkerRecoveryTestCase { + description: "invalid image pin retains a rolled-back launch observation", + image_id: Some("sha256:invalid"), + failure_event: "platform_launch_failed", + expected_pending_observations: 0, + expected_status: "failed", + }, + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("recovery test runtime"); + for test_case in test_cases { + runtime.block_on(async { + let mut control = start_control(&["server", "--port", "0", "--no-scheduler"]); + let url = api_ready(&mut control).await; + let (worker, baseline, _) = claim_worker_run(&control, test_case.image_id).await; + let connection = + rusqlite::Connection::open(control.directory.path().join(".sqlbuild/control.sqlite3")) + .expect("observe durable ownership"); + connection + .busy_timeout(Duration::from_secs(5)) + .expect("database contention timeout"); + connection.execute_batch(&format!( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00Z'; + CREATE TRIGGER reject_recovery_publication BEFORE INSERT ON recall_events + WHEN NEW.event_type='{}' BEGIN SELECT RAISE(ABORT, 'synthetic unavailable publication'); END;", test_case.failure_event + )).expect("expire the departed leader and interrupt durable publication"); + let mut replacement = start_recovery_scheduler(&control, &url, baseline.location_id); + wait_for_worker_log(&mut replacement, "Retrying worker shutdown publication").await; + let held: (i64, i64) = connection.query_row( + "SELECT (SELECT COUNT(*) FROM sqb_write_claims), (SELECT COUNT(*) FROM sqb_run_liveness)", + [], |row| Ok((row.get(0)?, row.get(1)?)), + ).expect("retained claim while publication is unavailable"); + assert_eq!(held, (1, 1), "{}", test_case.description); + for event_type in ["run_launch_failed", "platform_launch_failed"] { + let pending: i64 = connection.query_row("SELECT COUNT(*) FROM recall_events WHERE event_type=?1", [event_type], |row| row.get(0)).expect("pending source/platform pair"); + assert_eq!(pending, test_case.expected_pending_observations, "{}: {event_type}", test_case.description); + } + connection + .execute_batch("DROP TRIGGER reject_recovery_publication") + .expect("restore terminal publication"); + let exit = wait_for_worker_exit(&mut replacement).await; + assert!( + !exit.success(), + "the original publication failure remains visible" + ); + let finished: (String, i64, i64, i64) = connection.query_row( + "SELECT status, (SELECT COUNT(*) FROM sqb_write_claims), + (SELECT COUNT(*) FROM sqb_run_liveness), + (SELECT COUNT(*) FROM recall_events WHERE event_type='run_terminated') FROM sqb_runs WHERE run_id=?1", + [worker.identity.0.to_string()], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ).expect("terminal fact and released durable claims"); + assert_eq!( + finished, + (test_case.expected_status.into(), 0, 0, 1), + "{}", + test_case.description + ); + let (producer, payload, count): (String, String, i64) = connection.query_row( + "SELECT producer, payload, COUNT(*) OVER () FROM recall_events WHERE event_type='platform_launch_failed'", + [], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ).expect("durable launch observation from the real replacement scheduler"); + assert_eq!(producer, "platform"); + assert_eq!(count, 1, "{}", test_case.description); + let source_count: i64 = connection.query_row("SELECT COUNT(*) FROM recall_events WHERE event_type='run_launch_failed'", [], |row| row.get(0)).expect("retried source fact"); + assert_eq!(source_count, 1, "{}", test_case.description); + let event: serde_json::Value = serde_json::from_str(&payload).expect("typed event payload"); + assert_eq!(event["platform"]["kind"], "launch_failed"); + assert_eq!(event["platform"]["run_id"], worker.identity.0.to_string()); + assert_eq!(event["platform"]["claim_id"], worker.identity.1.to_string()); + assert_eq!(event["platform"]["code"], "docker_configuration_rejected"); + assert!( + std::fs::read_to_string(&replacement.stderr) + .expect("shutdown recovery log") + .contains("Worker shutdown recovered") + ); + }); + } +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs new file mode 100644 index 0000000000..0a6741cff7 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_types.rs @@ -0,0 +1,30 @@ +use crate::test_types::TestControl; + +pub(crate) struct LeaseCapture { + pub(crate) lease: Option< + tokio::sync::oneshot::Sender, + >, + pub(crate) shutdown: tokio::sync::watch::Sender, +} + +pub(crate) struct WorkerHeartbeatTestCase { + pub(crate) description: &'static str, + pub(crate) interruption: fn(WorkerControl, &mut TestControl), + pub(crate) expected_cancelled: bool, +} + +pub(crate) struct WorkerRecoveryTestCase { + pub(crate) description: &'static str, + pub(crate) image_id: Option<&'static str>, + pub(crate) failure_event: &'static str, + pub(crate) expected_pending_observations: i64, + pub(crate) expected_status: &'static str, +} + +pub(crate) struct WorkerControl { + pub(crate) store: sqlbuild_orchestration::store::models::ControlStore, + pub(crate) context: sqlbuild_orchestration::credentials::models::AuditContext, + pub(crate) lease: sqlbuild_orchestration::scheduler::models::SchedulerLease, + pub(crate) token_id: uuid::Uuid, + pub(crate) identity: (uuid::Uuid, uuid::Uuid), +} diff --git a/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs new file mode 100644 index 0000000000..5bb5be2014 --- /dev/null +++ b/crates/sqlbuild-cli/tests/native_commands/worker/test_worker.rs @@ -0,0 +1,8 @@ +#[path = "helpers.rs"] +mod helpers; +#[path = "test_heartbeat.rs"] +mod test_heartbeat; +#[path = "test_recovery.rs"] +mod test_recovery; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index f5c5246f56..7a68e32d8a 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -24,7 +24,7 @@ pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; /// The warehouse group each CLI command selects a target's default warehouse from. -pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 31] = [ +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("audit", Some("build")), ("build", Some("build")), ("check", Some("build")), @@ -50,6 +50,7 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 31] = [ ("format", None), ("init", None), ("lineage", None), + ("location", None), ("mv", None), ("playground", None), ("rename", None), diff --git a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs index 04e8d3084f..9eb5e8408f 100644 --- a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs +++ b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs @@ -43,6 +43,17 @@ pub fn format_debug_json(report: &DebugReport) -> String { ), ("providers".to_owned(), lines_value(&report.providers)), ("connection".to_owned(), lines_value(&report.connection)), + ( + "local_files".to_owned(), + JsonValue::Array( + report + .local_files + .iter() + .cloned() + .map(JsonValue::String) + .collect(), + ), + ), ]); let options: StdlibJsonOptions = StdlibJsonOptions { sort_keys: true, diff --git a/crates/sqlbuild-core/src/debug_report/models.rs b/crates/sqlbuild-core/src/debug_report/models.rs index c0ec5075af..144079f7d9 100644 --- a/crates/sqlbuild-core/src/debug_report/models.rs +++ b/crates/sqlbuild-core/src/debug_report/models.rs @@ -38,6 +38,8 @@ pub struct DebugReport { pub configuration: Vec, pub providers: Vec, pub connection: Vec, + /// Exact local database paths, not the shortened display values in connection lines. + pub local_files: Vec, } impl DebugReport { diff --git a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs index 623f516cd6..240efc55d5 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs @@ -2,6 +2,7 @@ use crate::debug_report::models::{ConnectionValue, DebugLine, DebugReport, Debug pub(super) fn sample_report() -> DebugReport { DebugReport { + local_files: vec!["/repo/warehouse.db".into()], runtime: vec![DebugLine::plain("sqlbuild version", "0.2.1")], configuration: vec![DebugLine::checked( "project file", diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs index 7a2122f2d7..5fb74ee026 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs @@ -18,7 +18,7 @@ fn given_debug_report_when_formatting_then_matches_python_outputs() { description: "renders diagnostics sections and json sections", report: sample_report(), expected_text: "\nSQLBuild Diagnostics\n\nRuntime:\n sqlbuild version: 0.2.1\n\nConfiguration:\n project file: /repo/sqlbuild_project.toml [OK found and valid]\n\nProviders:\n providers: 1 [OK discovered]\n marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n\nConnection:\n token: ****\n connection test: [ERROR authentication failed]\n query test: [SKIP connection failed]\n\n", - expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, + expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "local_files": ["/repo/warehouse.db"], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, expected_color_fragments: &[ "\x1b[34m\x1b[1mSQLBuild Diagnostics\x1b[0m", "\x1b[1mRuntime:\x1b[0m", diff --git a/crates/sqlbuild-core/src/identifier_text/main/mod.rs b/crates/sqlbuild-core/src/identifier_text/main/mod.rs index a29dc793b1..b9d09a0ef5 100644 --- a/crates/sqlbuild-core/src/identifier_text/main/mod.rs +++ b/crates/sqlbuild-core/src/identifier_text/main/mod.rs @@ -1 +1,2 @@ pub mod is_lowercase_identifier; +pub mod quote_identifier; diff --git a/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs b/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs new file mode 100644 index 0000000000..efd258cbc0 --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/main/quote_identifier.rs @@ -0,0 +1,6 @@ +//! Standard SQL double-quoted identifiers, including embedded quote escaping. + +#[must_use] +pub fn quote_identifier(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 36ee9d8e7e..bcdf9d0799 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -110,6 +110,7 @@ pub(crate) const SQL_PREVIEW_LENGTH: usize = 160; pub(crate) const EXECUTION_TITLE: &str = "Execution"; pub(crate) const LINEAGE_COMMAND: &str = "lineage"; pub(crate) const QUERY_COMMAND: &str = "query"; +pub(crate) const LOCATION_COMMAND: &str = "location"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_JSON_OPTION: &str = "--json"; pub(crate) const JSON_FORMAT: &str = "json"; diff --git a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs index 7a6c3bc531..1337d46807 100644 --- a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs +++ b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs @@ -1,7 +1,7 @@ //! Whether a command's stdout carries machine output, so human progress must use stderr. use crate::progress::constants::{ - DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, + DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, LOCATION_COMMAND, QUERY_COMMAND, }; use crate::progress::models::OutputModeFacts; @@ -11,6 +11,7 @@ pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { let format = |value: &Option| value.as_deref().unwrap_or_default().to_owned(); match facts.command.as_deref() { _ if facts.json => true, + Some(LOCATION_COMMAND) => true, Some(DBT_COMMAND) => facts.dbt_args.iter().any(|arg| arg == DBT_JSON_OPTION), Some(LINEAGE_COMMAND) => format(&facts.lineage_format) == JSON_FORMAT, Some(QUERY_COMMAND) => format(&facts.query_format) == JSON_FORMAT, diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index 096580ebab..86a7a4dcd5 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -2,6 +2,8 @@ /// Root of SQL model files. pub const MODELS_DIRECTORY: &str = "models"; +/// Root of code-location orchestration declarations, never imported by the scheduler. +pub const ORCHESTRATION_DIRECTORY: &str = "orchestration"; /// Suffix of SQL files. pub const SQL_FILE_SUFFIX: &str = ".sql"; /// The folder below a resource tree that holds grouped scoped declarations. diff --git a/crates/sqlbuild-discovery/src/python_paths/constants.rs b/crates/sqlbuild-discovery/src/python_paths/constants.rs index eacdb349f9..4483a9f780 100644 --- a/crates/sqlbuild-discovery/src/python_paths/constants.rs +++ b/crates/sqlbuild-discovery/src/python_paths/constants.rs @@ -1,7 +1,9 @@ //! The documented roots project Python may live under, and the directories the walk skips. +use crate::constants::ORCHESTRATION_DIRECTORY; + /// Top-level directories whose Python files are always supported. -pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 10] = [ +pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 11] = [ "adapters", "materializations", "providers", @@ -12,6 +14,7 @@ pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 10] = [ "enums", "constants", "event_exporters", + ORCHESTRATION_DIRECTORY, ]; /// `(root, language)` directory pairs whose Python files are supported. pub(crate) const LANGUAGE_PYTHON_ROOTS: [(&str, &str); 2] = diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs index 64ecd45aa6..9a4843ec44 100644 --- a/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs @@ -11,6 +11,8 @@ fn given_project_python_when_listing_unsupported_paths_then_only_undocumented_ro files: &[ "adapter.py", "python/orders.py", + "orchestration/jobs.py", + "orchestration/nested/calendar.py", "functions/python/f.py", "macros/m.py", "models/_macros/m.py", diff --git a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs index 7deafb99c8..51d567c4e1 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/user_code.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/user_code.rs @@ -64,6 +64,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::UserCode, }, + HostEntry { + name: "load_orchestration_module", + kind: HostKind::Function, + edge: HostEdge::UserCode, + }, HostEntry { name: "module_functions", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 5c8482411a..057a1d4203 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -140,6 +140,7 @@ pub enum UserCodeHost { LoadProviderModule, LoadPythonNodeModule, LoadSinkModule, + LoadOrchestrationModule, ModuleFunctions, ModuleProviderClasses, ParameterAnnotations, diff --git a/crates/sqlbuild-orchestration/Cargo.toml b/crates/sqlbuild-orchestration/Cargo.toml new file mode 100644 index 0000000000..24f97c2b34 --- /dev/null +++ b/crates/sqlbuild-orchestration/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "sqlbuild-orchestration" +version.workspace = true +description = "SQLBuild control-plane storage, snapshots, scheduling and run coordination." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +axum.workspace = true +chrono.workspace = true +chrono-tz.workspace = true +croner.workspace = true +flate2.workspace = true +getrandom.workspace = true +reqwest.workspace = true +rusqlite.workspace = true +semver.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlbuild-recall.workspace = true +sqlbuild-state.workspace = true +sqlx-core.workspace = true +sqlx-postgres.workspace = true +subtle.workspace = true +thiserror.workspace = true +tokio.workspace = true +utoipa.workspace = true +uuid.workspace = true + +[dev-dependencies] +proptest.workspace = true +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs new file mode 100644 index 0000000000..70c607a995 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication.rs @@ -0,0 +1,92 @@ +//! Authenticate before body buffering; scope checks remain in each owning endpoint. + +use axum::extract::{Request, State}; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::middleware::Next; +use axum::response::Response; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::ReadResource; +use crate::credentials::main::authenticate_token::authenticate_token; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn authenticate( + State(store): State, + mut request: Request, + next: Next, +) -> Result { + let unauthorized = || ApiError { + status: StatusCode::UNAUTHORIZED, + message: "valid bearer credential required".into(), + }; + let bearer = request + .headers() + .get(header::AUTHORIZATION) + .ok_or_else(unauthorized)? + .to_str() + .map_err(|_| unauthorized())? + .strip_prefix("Bearer ") + .ok_or_else(unauthorized)?; + let identity = authenticate_token(&store, bearer) + .await? + .ok_or_else(unauthorized)?; + request.extensions_mut().insert(identity); + let mut response = next.run(request).await; + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + Ok(response) +} + +pub(crate) fn require_location( + identity: &TokenIdentity, + location_id: Uuid, +) -> Result<(), ApiError> { + if identity.scope != (CredentialScope::Location { location_id }) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this location".into(), + }); + } + Ok(()) +} + +/// A guessed resource ID never widens the already authenticated location and target. +pub(crate) async fn require_resource( + store: &ControlStore, + (location_id, target): (Uuid, &str), + (resource, id): (ReadResource, Uuid), +) -> Result<(), ApiError> { + let (statement, missing) = match resource { + ReadResource::Run => ( + "SELECT run_id FROM sqb_runs WHERE run_id=$1 AND location_id=$2 AND target=$3", + "run not found", + ), + ReadResource::Trigger => ( + "SELECT trigger_id FROM sqb_triggers WHERE trigger_id=$1 AND location_id=$2 AND target=$3", + "trigger not found", + ), + }; + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + statement, + vec![ + Parameter::Id(id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + ], + ) + .await?; + commit(transaction).await?; + if rows.is_empty() { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: missing.into(), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/client.rs b/crates/sqlbuild-orchestration/src/api/_helpers/client.rs new file mode 100644 index 0000000000..98532b9054 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/client.rs @@ -0,0 +1,38 @@ +//! Each client runtime owns its pool; credentials and server response bodies stay out of diagnostics. + +use std::time::Duration; + +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; + +use crate::api::errors::ClientError; +use crate::api::models::ControlClient; + +pub(crate) fn http_client(authorization: HeaderValue) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, authorization); + Ok(reqwest::Client::builder() + .default_headers(headers) + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .build()?) +} + +impl ControlClient { + pub(crate) fn independent(&self) -> Result { + Ok(Self { + client: http_client(self.authorization.clone())?, + authorization: self.authorization.clone(), + base_url: self.base_url.clone(), + }) + } + + pub(crate) fn endpoint(&self, segments: &[&str]) -> Result { + let mut url = self.base_url.clone(); + url.path_segments_mut() + .map_err(|_| ClientError::Configuration("server URL cannot contain an API path"))? + .pop_if_empty() + .extend(segments.iter().copied()); + Ok(url) + } +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs b/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs new file mode 100644 index 0000000000..60a407233c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/location_health.rs @@ -0,0 +1,45 @@ +//! Scope both observations and reads before exposing any location history. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::platform::main::location_health::location_health; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealthObservation, LocationHealthReport}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/health", + params(("location_id" = Uuid, Path)), request_body = LocationHealthReport, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, + Json(report): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: report_location_health(&store, location_id, &report).await?, + })) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/health", + params(("location_id" = Uuid, Path)), + responses((status = 200, body = Vec), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn recent( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(location_health(&store, location_id).await?)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs b/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs new file mode 100644 index 0000000000..6e48859e78 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/location_work.rs @@ -0,0 +1,65 @@ +//! Locations may poll and complete only their own work; enqueueing stays a trusted-owner action. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::platform::main::claim_location_work::claim_location_work; +use crate::platform::main::complete_location_work::complete_location_work; +use crate::platform::main::location_work::location_work; +use crate::platform::models::{ + LocationWorkClaim, LocationWorkPoll, LocationWorkRecord, LocationWorkResult, +}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/work/poll", + params(("location_id" = Uuid, Path)), request_body = LocationWorkPoll, + responses((status = 200, body = Option), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn poll( + State(store): State, + Extension(identity): Extension, + Path(location_id): Path, + Json(poll): Json, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(claim_location_work(&store, location_id, &poll).await?)) +} + +#[utoipa::path(post, path = "/api/locations/{location_id}/work/{request_id}", + params(("location_id" = Uuid, Path), ("request_id" = Uuid, Path)), request_body = LocationWorkResult, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Work claim expired or was replaced")), security(("bearer" = [])))] +pub(crate) async fn complete( + State(store): State, + Extension(identity): Extension, + Path((location_id, request_id)): Path<(Uuid, Uuid)>, + Json(result): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: complete_location_work(&store, (location_id, request_id), &result).await?, + })) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/work/{request_id}", + params(("location_id" = Uuid, Path), ("request_id" = Uuid, Path)), + responses((status = 200, body = Option), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn result( + State(store): State, + Extension(identity): Extension, + Path((location_id, request_id)): Path<(Uuid, Uuid)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(location_work(&store, request_id).await?.filter( + |record| record.request_json.location_id == location_id, + ))) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs new file mode 100644 index 0000000000..94872aa28d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod authentication; +pub(crate) mod client; +pub(crate) mod location_health; +pub(crate) mod location_work; +pub(crate) mod run_reads; +pub(crate) mod runs; +pub(crate) mod snapshots; +pub(crate) mod trigger_reads; +pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs new file mode 100644 index 0000000000..06ee0574ab --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/run_reads.rs @@ -0,0 +1,132 @@ +//! Location credentials can observe only their own runs; they cannot request or mutate work. + +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::{require_location, require_resource}; +use crate::api::errors::ApiError; +use crate::api::models::{ReadQuery, ReadResource, RunRecallPath, RunRecallQuery}; +use crate::credentials::models::TokenIdentity; +use crate::runs::main::list_run_nodes::list_run_nodes; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunNodeRecord, RunRecord}; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::list_recall_invocations::list_recall_invocations; +use crate::store::main::query::query; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("limit" = Option, Query, description = "Maximum recent runs; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn recent( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json( + list_runs(&store, (location_id, &target), options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path)), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Run not found")), + security(("bearer" = [])))] +pub(crate) async fn nodes( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + Ok(Json(list_run_nodes(&store, run_id).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum invocation namespaces; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Run not found")), + security(("bearer" = [])))] +pub(crate) async fn invocations( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + let namespaces = run_namespaces(&store, run_id).await?; + Ok(Json( + list_recall_invocations(&store, &namespaces, options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), + ("view" = String, Path), ("invocation" = String, Query, description = "Exact scheduler or worker invocation ID"), + ("limit" = Option, Query, description = "Maximum rows; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 400, description = "Unknown Recall view"), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 404, description = "Run or invocation not found")), security(("bearer" = [])))] +pub(crate) async fn recall( + State(store): State, + Extension(identity): Extension, + Path(RunRecallPath { + location_id, + target, + run_id, + view, + }): Path, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource(&store, (location_id, &target), (ReadResource::Run, run_id)).await?; + let namespaces = run_namespaces(&store, run_id).await?; + if !namespaces.iter().any(|namespace| { + options.invocation == *namespace || options.invocation.starts_with(&format!("{namespace}/")) + }) { + return Err(ApiError { + status: StatusCode::NOT_FOUND, + message: "run invocation not found".into(), + }); + } + Ok(Json( + read_recall_view( + &store, + (&view, &options.invocation), + options.limit.unwrap_or(100), + ) + .await?, + )) +} + +/// Immutable claim evidence remains available after terminal publication removes liveness. +async fn run_namespaces(store: &ControlStore, run_id: Uuid) -> Result, ApiError> { + let scheduler = format!("scheduler/{run_id}"); + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, claims) = query( + transaction, + "SELECT payload FROM recall_events WHERE invocation_id=$1 AND event_type='run_claimed' ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(scheduler.clone())], + ).await?; + commit(transaction).await?; + let mut namespaces = vec![scheduler]; + if let Some(claim_id) = claims + .first() + .and_then(|row| row["payload"]["claim"]["claim_id"].as_str()) + { + namespaces.push(format!("worker/{claim_id}")); + } + Ok(namespaces) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs new file mode 100644 index 0000000000..3762a428bf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/runs.rs @@ -0,0 +1,124 @@ +//! Worker reports are bound to the authenticated run and claim, never a caller-supplied actor. + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use sqlbuild_recall::records::models::OutputChunk; +use sqlbuild_recall::store::models::InsertOutcome; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::{HeartbeatResponse, MutationResponse}; +use crate::credentials::models::{CredentialScope, TokenIdentity}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::publish_run_output::publish_run_output; +use crate::runs::main::publish_run_recall::publish_run_recall; +use crate::runs::main::run_execution::run_execution; +use crate::runs::models::{ + OutputPublication, RecallBatch, RunAuthority, RunChangeRequest, RunExecution, +}; +use crate::store::models::ControlStore; + +#[utoipa::path(get, path = "/api/runs/{run_id}/execution", params(("run_id" = Uuid, Path)), + responses((status = 200, body = RunExecution), (status = 401, description = "Inactive claim or invalid credential"), + (status = 403, description = "Run not authorized")), security(("bearer" = [])))] +pub(crate) async fn execution( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + let execution = run_execution(&store, (run_id, claim_id)) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::UNAUTHORIZED, + message: "run claim is no longer active".into(), + })?; + Ok(Json(execution)) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/events", params(("run_id" = Uuid, Path)), + request_body = RunChangeRequest, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(request): Json, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + if request.run_id != run_id || request.claim_id != Some(claim_id) { + return Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "report does not match the authenticated run and claim".into(), + }); + } + let applied = change_run(&store, RunAuthority::Worker { run_id, claim_id }, &request).await?; + Ok(Json(MutationResponse { applied })) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/heartbeat", params(("run_id" = Uuid, Path)), + responses((status = 200, body = HeartbeatResponse), (status = 401, description = "Invalid credential"), + (status = 403, description = "Run not authorized")), security(("bearer" = [])))] +pub(crate) async fn heartbeat( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json(HeartbeatResponse { + cancel_requested: heartbeat_run(&store, (run_id, claim_id)).await?, + })) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/output", params(("run_id" = Uuid, Path)), + request_body = Vec, + responses((status = 200, body = OutputPublication), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn output( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(chunks): Json>, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json( + publish_run_output(&store, (run_id, claim_id), &chunks).await?, + )) +} + +#[utoipa::path(post, path = "/api/runs/{run_id}/recall", params(("run_id" = Uuid, Path)), + request_body = RecallBatch, + responses((status = 200, body = InsertOutcome), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Run not authorized")), + security(("bearer" = [])))] +pub(crate) async fn recall( + State(store): State, + Extension(identity): Extension, + Path(run_id): Path, + Json(batch): Json, +) -> Result, ApiError> { + let claim_id = require_worker(&identity, run_id)?; + Ok(Json( + publish_run_recall(&store, (run_id, claim_id), &batch).await?, + )) +} + +fn require_worker(identity: &TokenIdentity, run_id: Uuid) -> Result { + if let CredentialScope::Run { + run_id: permitted, + claim_id, + } = identity.scope + && permitted == run_id + { + return Ok(claim_id); + } + Err(ApiError { + status: StatusCode::FORBIDDEN, + message: "credential does not authorize this run".into(), + }) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs new file mode 100644 index 0000000000..0b6f56f2af --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/snapshots.rs @@ -0,0 +1,77 @@ +//! A location credential publishes and reads only its own location's snapshots. + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::credentials::models::TokenIdentity; +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::current_snapshot_record::current_snapshot_record; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{CurrentSnapshot, SnapshotInput, SnapshotRecord}; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/snapshots", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + request_body = SnapshotInput, + responses((status = 200, body = SnapshotRecord), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized"), + (status = 409, description = "Stale generation")), security(("bearer" = [])))] +pub(crate) async fn publish( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(input): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + if input.location_id != location_id || input.target != target { + return Err(ApiError { + status: StatusCode::BAD_REQUEST, + message: "snapshot does not match the requested location and target".into(), + }); + } + Ok(Json(publish_snapshot(&store, &input).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/snapshots", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = CurrentSnapshot), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "No successful snapshot")), + security(("bearer" = [])))] +pub(crate) async fn current( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + let current = current_snapshot(&store, location_id, &target) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::NOT_FOUND, + message: "no successful snapshot".into(), + })?; + Ok(Json(current)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/snapshots/metadata", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = SnapshotRecord), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "No successful snapshot")), + security(("bearer" = [])))] +pub(crate) async fn metadata( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + let record = current_snapshot_record(&store, location_id, &target) + .await? + .ok_or_else(|| ApiError { + status: StatusCode::NOT_FOUND, + message: "no successful snapshot".into(), + })?; + Ok(Json(record)) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs new file mode 100644 index 0000000000..bcc5fd5bdd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/trigger_reads.rs @@ -0,0 +1,77 @@ +//! Trigger definitions and immutable history are visible only within the authenticated location. + +use axum::extract::{Path, Query, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::{require_location, require_resource}; +use crate::api::errors::ApiError; +use crate::api::models::{ReadQuery, ReadResource}; +use crate::credentials::models::TokenIdentity; +use crate::declarations::main::list_trigger_events::list_trigger_events; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChangeRecord, TriggerRecord}; +use crate::scheduler::main::list_ticks::list_ticks; +use crate::scheduler::models::TickRecord; +use crate::store::models::ControlStore; + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers", + params(("location_id" = Uuid, Path), ("target" = String, Path)), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized")), security(("bearer" = [])))] +pub(crate) async fn current( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(list_triggers(&store, location_id, &target).await?)) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("trigger_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum events; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Trigger not found")), + security(("bearer" = [])))] +pub(crate) async fn events( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, trigger_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource( + &store, + (location_id, &target), + (ReadResource::Trigger, trigger_id), + ) + .await?; + Ok(Json( + list_trigger_events(&store, trigger_id, options.limit.unwrap_or(100)).await?, + )) +} + +#[utoipa::path(get, path = "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("trigger_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum recent ticks; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec), (status = 401, description = "Invalid credential"), + (status = 403, description = "Location not authorized"), (status = 404, description = "Trigger not found")), + security(("bearer" = [])))] +pub(crate) async fn ticks( + State(store): State, + Extension(identity): Extension, + Path((location_id, target, trigger_id)): Path<(Uuid, String, Uuid)>, + Query(options): Query, +) -> Result>, ApiError> { + require_location(&identity, location_id)?; + require_resource( + &store, + (location_id, &target), + (ReadResource::Trigger, trigger_id), + ) + .await?; + Ok(Json( + list_ticks(&store, trigger_id, options.limit.unwrap_or(100)).await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs new file mode 100644 index 0000000000..5ddb1f2001 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/warehouse_metadata.rs @@ -0,0 +1,30 @@ +//! An authenticated location can mirror only its own registered reader's target. + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::_helpers::authentication::require_location; +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::TokenIdentity; +use crate::store::models::ControlStore; +use crate::warehouse_metadata::main::ingest_source_freshness::ingest_source_freshness; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +#[utoipa::path(post, path = "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", + params(("location_id" = Uuid, Path), ("target" = String, Path)), request_body = SourceFreshnessBatch, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Invalid credential"), (status = 403, description = "Location not authorized")), + security(("bearer" = [])))] +pub(crate) async fn report( + State(store): State, + Extension(identity): Extension, + Path((location_id, target)): Path<(Uuid, String)>, + Json(batch): Json, +) -> Result, ApiError> { + require_location(&identity, location_id)?; + Ok(Json(MutationResponse { + applied: ingest_source_freshness(&store, (location_id, &target), &batch).await?, + })) +} diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs new file mode 100644 index 0000000000..d3d0c93f8d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -0,0 +1,3 @@ +//! Shared HTTP transport bounds for the server and native clients. + +pub(crate) const BODY_BYTES: usize = 64 * 1024 * 1024; diff --git a/crates/sqlbuild-orchestration/src/api/errors.rs b/crates/sqlbuild-orchestration/src/api/errors.rs new file mode 100644 index 0000000000..ea9da3591d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/errors.rs @@ -0,0 +1,64 @@ +//! HTTP errors never return database diagnostics, connection strings, or credentials. + +use axum::Json; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; + +use crate::api::models::ApiErrorBody; +use crate::store::errors::ControlError; + +#[derive(Debug, thiserror::Error)] +pub enum ClientError { + #[error("invalid API connection: {0}")] + Configuration(&'static str), + #[error("API request failed: {0}")] + Transport(#[from] reqwest::Error), + #[error("API rejected the request with HTTP {0}")] + Rejected(reqwest::StatusCode), +} + +#[derive(Debug)] +pub struct ApiError { + pub(crate) status: StatusCode, + pub(crate) message: String, +} + +impl From for ApiError { + fn from(error: ControlError) -> Self { + match error { + ControlError::Invalid(message) => Self { + status: StatusCode::BAD_REQUEST, + message, + }, + ControlError::StaleGeneration | ControlError::Fenced => Self { + status: StatusCode::CONFLICT, + message: error.to_string(), + }, + _ => Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: "control-store operation failed".into(), + }, + } + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let mut response = ( + self.status, + Json(ApiErrorBody { + error: self.message, + }), + ) + .into_response(); + response + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + if self.status == StatusCode::UNAUTHORIZED { + response + .headers_mut() + .insert(header::WWW_AUTHENTICATE, HeaderValue::from_static("Bearer")); + } + response + } +} diff --git a/crates/sqlbuild-orchestration/src/api/main/client_response.rs b/crates/sqlbuild-orchestration/src/api/main/client_response.rs new file mode 100644 index 0000000000..2eb643424f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/client_response.rs @@ -0,0 +1,15 @@ +//! Share bounded JSON transport without exposing server bodies or credential-bearing URLs. + +use serde::de::DeserializeOwned; + +use crate::api::errors::ClientError; + +pub(crate) async fn client_response( + request: reqwest::RequestBuilder, +) -> Result { + let response = request.send().await.map_err(reqwest::Error::without_url)?; + if !response.status().is_success() { + return Err(ClientError::Rejected(response.status())); + } + Ok(response.json().await.map_err(reqwest::Error::without_url)?) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs new file mode 100644 index 0000000000..c0b5258123 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs @@ -0,0 +1,31 @@ +//! Validate the API origin before attaching a sensitive bearer credential. + +use reqwest::header::HeaderValue; + +use crate::api::_helpers::client::http_client; +use crate::api::errors::ClientError; +use crate::api::models::ControlClient; + +pub(crate) fn connect_client(server_url: &str, bearer: &str) -> Result { + let base_url = reqwest::Url::parse(server_url) + .map_err(|_| ClientError::Configuration("server URL must be an absolute HTTP URL"))?; + if !matches!(base_url.scheme(), "http" | "https") + || base_url.host_str().is_none() + || !base_url.username().is_empty() + || base_url.password().is_some() + || base_url.query().is_some() + || base_url.fragment().is_some() + { + return Err(ClientError::Configuration( + "server URL must use HTTP or HTTPS without credentials, query or fragment", + )); + } + let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}")) + .map_err(|_| ClientError::Configuration("bearer is not a valid HTTP credential"))?; + authorization.set_sensitive(true); + Ok(ControlClient { + client: http_client(authorization.clone())?, + authorization, + base_url, + }) +} diff --git a/crates/sqlbuild-orchestration/src/api/main/mod.rs b/crates/sqlbuild-orchestration/src/api/main/mod.rs new file mode 100644 index 0000000000..f445706317 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod client_response; +pub(crate) mod connect_client; +pub mod openapi_document; +pub mod serve_api; diff --git a/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs new file mode 100644 index 0000000000..597993eda8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/openapi_document.rs @@ -0,0 +1,19 @@ +//! Generate the HTTP contract from the request/response types used by the actual endpoints. + +use utoipa::OpenApi; +use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityScheme}; + +use crate::api::models::ControlApi; + +#[must_use] +pub fn openapi_document() -> utoipa::openapi::OpenApi { + let mut document = ControlApi::openapi(); + document + .components + .get_or_insert_with(Default::default) + .add_security_scheme( + "bearer", + SecurityScheme::Http(HttpBuilder::new().scheme(HttpAuthScheme::Bearer).build()), + ); + document +} diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs new file mode 100644 index 0000000000..dd000306a9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -0,0 +1,93 @@ +//! One native server owns migrations and all storage writes; locations and workers speak HTTP. + +use axum::extract::DefaultBodyLimit; +use axum::routing::{get, post}; +use axum::{Json, Router, middleware}; + +use crate::api::_helpers::authentication::authenticate; +use crate::api::_helpers::{ + location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, +}; +use crate::api::constants::BODY_BYTES; +use crate::api::main::openapi_document::openapi_document; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; + +pub async fn serve_api( + store: ControlStore, + listener: tokio::net::TcpListener, + shutdown: impl Future + Send + 'static, +) -> Result<(), ControlError> { + migrate_store(&store).await?; + let protected = Router::new() + .route( + "/api/locations/{location_id}/work/poll", + post(location_work::poll), + ) + .route( + "/api/locations/{location_id}/work/{request_id}", + get(location_work::result).post(location_work::complete), + ) + .route( + "/api/locations/{location_id}/health", + get(location_health::recent).post(location_health::report), + ) + .route( + "/api/locations/{location_id}/targets/{target}/metadata/source-freshness", + post(warehouse_metadata::report), + ) + .route( + "/api/locations/{location_id}/targets/{target}/snapshots", + get(snapshots::current).post(snapshots::publish), + ) + .route( + "/api/locations/{location_id}/targets/{target}/snapshots/metadata", + get(snapshots::metadata), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs", + get(run_reads::recent), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + get(run_reads::nodes), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + get(run_reads::invocations), + ) + .route( + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + get(run_reads::recall), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers", + get(trigger_reads::current), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + get(trigger_reads::events), + ) + .route( + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + get(trigger_reads::ticks), + ) + .route("/api/runs/{run_id}/execution", get(runs::execution)) + .route("/api/runs/{run_id}/events", post(runs::report)) + .route("/api/runs/{run_id}/heartbeat", post(runs::heartbeat)) + .route("/api/runs/{run_id}/output", post(runs::output)) + .route("/api/runs/{run_id}/recall", post(runs::recall)) + .layer(DefaultBodyLimit::max(BODY_BYTES)) + .route_layer(middleware::from_fn_with_state(store.clone(), authenticate)); + let application = protected + .route( + "/api/openapi.json", + get(|| async { Json(openapi_document()) }), + ) + .with_state(store); + axum::serve(listener, application) + .with_graceful_shutdown(shutdown) + .await?; + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/api/mod.rs b/crates/sqlbuild-orchestration/src/api/mod.rs new file mode 100644 index 0000000000..524d320a8c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/mod.rs @@ -0,0 +1,11 @@ +//! Python-free HTTP endpoints; locations and workers have no direct database access. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/api/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs new file mode 100644 index 0000000000..9668db92c1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -0,0 +1,82 @@ +//! Typed HTTP responses shared with the generated OpenAPI contract. + +use serde::{Deserialize, Serialize}; + +use crate::api::_helpers::{ + location_health, location_work, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, +}; + +#[derive(Debug, utoipa::OpenApi)] +#[openapi(paths( + location_health::report, + location_health::recent, + location_work::poll, + location_work::complete, + location_work::result, + snapshots::publish, + snapshots::current, + snapshots::metadata, + runs::execution, + runs::report, + runs::heartbeat, + runs::output, + runs::recall, + run_reads::recent, + run_reads::nodes, + run_reads::invocations, + run_reads::recall, + trigger_reads::current, + trigger_reads::events, + trigger_reads::ticks, + warehouse_metadata::report +))] +pub(crate) struct ControlApi; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ReadQuery { + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RunRecallQuery { + pub(crate) invocation: String, + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct RunRecallPath { + pub(crate) location_id: uuid::Uuid, + pub(crate) target: String, + pub(crate) run_id: uuid::Uuid, + pub(crate) view: String, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum ReadResource { + Run, + Trigger, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct MutationResponse { + pub applied: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct HeartbeatResponse { + pub cancel_requested: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ApiErrorBody { + pub error: String, +} + +#[derive(Clone, Debug)] +pub(crate) struct ControlClient { + pub(crate) client: reqwest::Client, + pub(crate) authorization: reqwest::header::HeaderValue, + pub(crate) base_url: reqwest::Url, +} diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs new file mode 100644 index 0000000000..ae89a5d233 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod scopes; diff --git a/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs new file mode 100644 index 0000000000..151288c4c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/_helpers/scopes.rs @@ -0,0 +1,40 @@ +//! Service credentials stop being valid with their live principal, location, or run claim. + +use crate::credentials::models::CredentialScope; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Bound values in principal/location/run/claim column order for the credential insert. +pub(crate) fn scope_parameters(scope: CredentialScope) -> Vec { + match scope { + CredentialScope::Principal { principal_id } => [Some(principal_id), None, None, None], + CredentialScope::Location { location_id } => [None, Some(location_id), None, None], + CredentialScope::Run { run_id, claim_id } => [None, None, Some(run_id), Some(claim_id)], + } + .into_iter() + .map(Parameter::OptionalId) + .collect() +} + +pub(crate) async fn scope_is_active( + transaction: ControlTransaction, + scope: CredentialScope, +) -> Result<(ControlTransaction, bool), ControlError> { + let (sql, parameters) = match scope { + CredentialScope::Principal { principal_id } => ( + "SELECT principal_id FROM sqb_principals WHERE principal_id=$1 AND disabled_at IS NULL", + vec![Parameter::Id(principal_id)], + ), + CredentialScope::Location { location_id } => ( + "SELECT location_id FROM sqb_locations WHERE location_id=$1 AND archived_at IS NULL", + vec![Parameter::Id(location_id)], + ), + CredentialScope::Run { run_id, claim_id } => ( + "SELECT run_id FROM sqb_run_liveness WHERE run_id=$1 AND worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ), + }; + let (transaction, rows) = query(transaction, sql, parameters).await?; + Ok((transaction, !rows.is_empty())) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/constants.rs b/crates/sqlbuild-orchestration/src/credentials/constants.rs new file mode 100644 index 0000000000..ce3675812c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/constants.rs @@ -0,0 +1,4 @@ +//! Credential secrets use 256 bits of entropy, represented as hexadecimal in bearer strings. + +pub(crate) const TOKEN_SECRET_BYTES: usize = 32; +pub(crate) const TOKEN_SECRET_HEX_LENGTH: usize = TOKEN_SECRET_BYTES * 2; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs b/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs new file mode 100644 index 0000000000..30b2c7362d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/append_audit.rs @@ -0,0 +1,31 @@ +//! Security changes and their audit records commit together, without bearer material. + +use serde_json::Value; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn append_audit( + transaction: ControlTransaction, + context: &AuditContext, + (event_id, action): (Uuid, &str), + (scope, before, after): (Value, Value, Value), +) -> Result { + if context.actor.trim().is_empty() || context.granting_role.trim().is_empty() { + return Err(ControlError::Invalid( + "audit actor and granting role are required".into(), + )); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_security_audit (event_id, occurred_at, actor, action, granting_role, request_id, ip_address, scope_json, before_json, after_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)", + vec![Parameter::Id(event_id), Parameter::Time(now), Parameter::Text(context.actor.clone()), + Parameter::Text(action.into()), Parameter::Text(context.granting_role.clone()), Parameter::Id(context.request_id), + Parameter::OptionalText(context.ip_address.map(|value| value.to_string())), + Parameter::Json(scope), Parameter::Json(before), Parameter::Json(after)]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs new file mode 100644 index 0000000000..4349c0d08b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/authenticate_token.rs @@ -0,0 +1,59 @@ +//! Every request sees revocation and current scope state, including across server replicas. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use subtle::ConstantTimeEq; +use uuid::Uuid; + +use crate::credentials::_helpers::scopes::scope_is_active; +use crate::credentials::constants::TOKEN_SECRET_HEX_LENGTH; +use crate::credentials::models::{StoredToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Invalid, revoked, expired and inactive-scope credentials are indistinguishable to callers. +pub async fn authenticate_token( + store: &ControlStore, + bearer: &str, +) -> Result, ControlError> { + let Some((id, secret)) = bearer.split_once('.') else { + return Ok(None); + }; + if secret.len() != TOKEN_SECRET_HEX_LENGTH + || !secret.bytes().all(|value| value.is_ascii_hexdigit()) + { + return Ok(None); + } + let Ok(id) = Uuid::parse_str(id) else { + return Ok(None); + }; + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query( + transaction, + "SELECT token_id, scope_json, secret_hash, expires_at FROM sqb_access_tokens \ + WHERE token_id=$1 AND revoked_at IS NULL AND expires_at>$2", + vec![Parameter::Id(id), Parameter::Time(now)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let token: StoredToken = serde_json::from_value(serde_json::json!(row)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let digest = sha256_hex(bearer.as_bytes()); + if !bool::from(digest.as_bytes().ct_eq(token.secret_hash.as_bytes())) { + commit(transaction).await?; + return Ok(None); + } + let (transaction, active) = scope_is_active(transaction, token.scope_json).await?; + commit(transaction).await?; + Ok(active.then_some(TokenIdentity { + token_id: token.token_id, + scope: token.scope_json, + expires_at: token.expires_at, + })) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs new file mode 100644 index 0000000000..a046c16405 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_location.rs @@ -0,0 +1,44 @@ +//! Create a server-owned location identity independent of its mutable label. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Register a new location; names are labels, never credentials or stable identifiers. +pub async fn create_location( + store: &ControlStore, + name: &str, + context: &AuditContext, +) -> Result { + if name.trim().is_empty() { + return Err(ControlError::Invalid("location name is required".into())); + } + let mut transaction = begin(store, TransactionMode::Write).await?; + let id = Uuid::now_v7(); + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_locations (location_id, name, created_by, created_at) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(id), Parameter::Text(name.into()), Parameter::Text(context.actor.clone()), Parameter::Time(now)]).await?; + commit( + append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "location.created"), + ( + json!({"location_id": id}), + json!(null), + json!({"name": name}), + ), + ) + .await?, + ) + .await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs new file mode 100644 index 0000000000..4b85936350 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/create_principal.rs @@ -0,0 +1,55 @@ +//! Server-owned people and service accounts; creating one never grants a role implicitly. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, PrincipalKind}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn create_principal( + store: &ControlStore, + context: &AuditContext, + (name, kind): (&str, PrincipalKind), +) -> Result { + if name.trim().is_empty() { + return Err(ControlError::Invalid("principal name is required".into())); + } + let transaction = begin(store, TransactionMode::Write).await?; + let id = Uuid::now_v7(); + let now = transaction.now; + let kind = match kind { + PrincipalKind::Person => "person", + PrincipalKind::ServiceAccount => "service_account", + }; + let (transaction, _) = query( + transaction, + "INSERT INTO sqb_principals (principal_id, name, kind, created_at) VALUES ($1,$2,$3,$4)", + vec![ + Parameter::Id(id), + Parameter::Text(name.into()), + Parameter::Text(kind.into()), + Parameter::Time(now), + ], + ) + .await?; + commit( + append_audit( + transaction, + context, + (Uuid::new_v5(&id, b"created"), "principal.created"), + ( + json!({"principal_id": id}), + json!(null), + json!({"name": name, "kind": kind}), + ), + ) + .await?, + ) + .await?; + Ok(id) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs new file mode 100644 index 0000000000..090bf85c66 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/issue_token.rs @@ -0,0 +1,78 @@ +//! Random bearer secrets are returned once; only their hashes and bounded scopes are durable. + +use std::fmt::Write; + +use chrono::{DateTime, SubsecRound, Utc}; +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::credentials::_helpers::scopes::{scope_is_active, scope_parameters}; +use crate::credentials::constants::TOKEN_SECRET_BYTES; +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::{AuditContext, CredentialScope, IssuedToken, TokenIdentity}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The caller authorizes token management before entering this storage operation. +pub async fn issue_token( + store: &ControlStore, + context: &AuditContext, + scope: CredentialScope, + expires_at: DateTime, +) -> Result { + let expires_at = expires_at.trunc_subsecs(6); + let transaction = begin(store, TransactionMode::Write).await?; + if expires_at <= transaction.now { + return Err(ControlError::Invalid( + "token expiry must be in the future".into(), + )); + } + let (transaction, active) = scope_is_active(transaction, scope).await?; + if !active { + return Err(ControlError::Invalid("token scope is not active".into())); + } + let token_id = Uuid::now_v7(); + let mut random = [0_u8; TOKEN_SECRET_BYTES]; + getrandom::fill(&mut random) + .map_err(|_| ControlError::Invalid("could not generate a secure token".into()))?; + let mut bearer = format!("{token_id}."); + for byte in random { + write!(bearer, "{byte:02x}") + .map_err(|_| ControlError::Invalid("could not format a secure token".into()))?; + } + let now = transaction.now; + let mut parameters: Vec = vec![ + Parameter::Id(token_id), + Parameter::Text(sha256_hex(bearer.as_bytes())), + ]; + parameters.extend(scope_parameters(scope)); + parameters.extend([ + Parameter::Json(json!(scope)), + Parameter::Time(now), + Parameter::Time(expires_at), + ]); + let (transaction, _) = query(transaction, + "INSERT INTO sqb_access_tokens (token_id, secret_hash, principal_id, location_id, run_id, claim_id, scope_json, created_at, expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + parameters).await?; + let identity = TokenIdentity { + token_id, + scope, + expires_at, + }; + commit( + append_audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"issued"), "token.issued"), + (json!(scope), json!(null), json!(identity)), + ) + .await?, + ) + .await?; + Ok(IssuedToken { identity, bearer }) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/main/mod.rs b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs new file mode 100644 index 0000000000..bde23d947a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod append_audit; +pub mod authenticate_token; +pub mod create_location; +pub mod create_principal; +pub mod issue_token; +pub mod revoke_token; diff --git a/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs new file mode 100644 index 0000000000..a7366783ae --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/main/revoke_token.rs @@ -0,0 +1,48 @@ +//! Revocation and its security audit are one transaction, without a per-process token cache. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Repeating a revocation does not create a second audit entry or rewrite its actor. +pub async fn revoke_token( + store: &ControlStore, + context: &AuditContext, + token_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "UPDATE sqb_access_tokens SET revoked_at=$1 WHERE token_id=$2 AND revoked_at IS NULL RETURNING scope_json", + vec![Parameter::Time(now), Parameter::Id(token_id)]).await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(false); + }; + let scope = row + .get("scope_json") + .cloned() + .ok_or_else(|| ControlError::Invalid("stored token scope is missing".into()))?; + commit( + append_audit( + transaction, + context, + (Uuid::new_v5(&token_id, b"revoked"), "token.revoked"), + ( + scope, + json!({"token_id": token_id, "revoked": false}), + json!({"token_id": token_id, "revoked": true}), + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/credentials/mod.rs b/crates/sqlbuild-orchestration/src/credentials/mod.rs new file mode 100644 index 0000000000..acde203166 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/mod.rs @@ -0,0 +1,10 @@ +//! Hashed, revocable credentials scoped to one principal, location, or claimed run. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/credentials/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs new file mode 100644 index 0000000000..f8f7d956f1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -0,0 +1,63 @@ +//! Credentials identify a caller; principal permissions are a separate authorization decision. + +use std::net::IpAddr; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PrincipalKind { + Person, + ServiceAccount, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CredentialScope { + Principal { principal_id: Uuid }, + Location { location_id: Uuid }, + Run { run_id: Uuid, claim_id: Uuid }, +} + +/// Server-derived request metadata, never copied from an action's JSON body. +#[derive(Clone, Debug)] +pub struct AuditContext { + pub actor: String, + pub granting_role: String, + pub request_id: Uuid, + pub ip_address: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TokenIdentity { + pub token_id: Uuid, + pub scope: CredentialScope, + pub expires_at: DateTime, +} + +/// The bearer is returned once. Debug output must never contain it. +#[derive(Serialize)] +pub struct IssuedToken { + pub identity: TokenIdentity, + pub bearer: String, +} + +impl std::fmt::Debug for IssuedToken { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("IssuedToken") + .field("identity", &self.identity) + .field("bearer", &"[redacted]") + .finish() + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredToken { + pub(crate) token_id: Uuid, + pub(crate) scope_json: CredentialScope, + pub(crate) secret_hash: String, + pub(crate) expires_at: DateTime, +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs new file mode 100644 index 0000000000..112ceab0af --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/documents.rs @@ -0,0 +1,48 @@ +//! Decode the same immutable declaration documents at publication and execution boundaries. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::declarations::models::Definitions; +use crate::store::errors::ControlError; + +fn document( + documents: &BTreeMap, + name: &str, +) -> Result, ControlError> { + documents.get(name).map_or_else( + || Ok(BTreeMap::new()), + |value| { + serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(format!("{name}: {error}"))) + }, + ) +} + +pub(crate) fn decode_definitions( + documents: &BTreeMap, +) -> Result { + let mut definitions = Definitions { + jobs: document(documents, "jobs")?, + groups: document(documents, "groups")?, + ..Definitions::default() + }; + for (name, value) in document::(documents, "triggers")? { + if value.get("when").is_some() { + definitions.sensors.insert( + name, + serde_json::from_value(value) + .map_err(|error| ControlError::Invalid(format!("triggers: {error}")))?, + ); + } else { + definitions.triggers.insert( + name, + serde_json::from_value(value) + .map_err(|error| ControlError::Invalid(format!("triggers: {error}")))?, + ); + } + } + Ok(definitions) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs new file mode 100644 index 0000000000..75c383f6bc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod documents; +pub(crate) mod operator_state; +pub(crate) mod trigger_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs new file mode 100644 index 0000000000..b07ce5e73b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/operator_state.rs @@ -0,0 +1,81 @@ +//! Apply the operator's projection and return the before/after audit images in the same transaction. + +use serde_json::{Value, json}; + +use crate::declarations::_helpers::trigger_state::trigger_timing; +use crate::declarations::models::{ + EditableTriggerState, TriggerChange, TriggerChangeRequest, TriggerDeclaration, TriggerStatus, +}; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::main::reset_sensor_cursor::reset_sensor_cursor; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn apply_operator_change( + transaction: ControlTransaction, + mut state: EditableTriggerState, + request: &TriggerChangeRequest, +) -> Result<(ControlTransaction, (Value, Value)), ControlError> { + let (mut transaction, cursor) = match &state.definition_json { + TriggerDeclaration::Sensor(_) => { + if matches!(request.change, TriggerChange::SetCursor { .. }) { + return Err(ControlError::Invalid( + "sensor cursors use storage order; use reset-cursor to replay retained events" + .into(), + )); + } + let (transaction, cursor) = read_sensor_cursor(transaction, request.trigger_id).await?; + (transaction, Some(cursor)) + } + TriggerDeclaration::Schedule(_) => (transaction, None), + }; + let mut before = + json!({"status_override": state.status_override, "last_tick_for": state.last_tick_for}); + if let Some(cursor) = &cursor { + before["cursor"] = json!(cursor); + } + match request.change { + TriggerChange::Enable => state.status_override = Some(TriggerStatus::Running), + TriggerChange::Pause => state.status_override = Some(TriggerStatus::Paused), + TriggerChange::ResetDefault => state.status_override = None, + TriggerChange::SetCursor { time } => state.last_tick_for = Some(time), + TriggerChange::ResetCursor => state.last_tick_for = None, + } + let (status, next) = match &state.definition_json { + TriggerDeclaration::Schedule(definition) => trigger_timing( + definition, + state.started_at, + state.last_tick_for, + state.status_override, + )?, + TriggerDeclaration::Sensor(definition) => { + if matches!(request.change, TriggerChange::ResetCursor) { + transaction = + reset_sensor_cursor(transaction, (request.trigger_id, request.event_id)) + .await?; + } + let status = state + .status_override + .unwrap_or(definition.default_status.projection()); + ( + status, + (status == TriggerStatus::Running).then_some(transaction.now), + ) + } + }; + let mut after = json!({"status_override": state.status_override, "last_tick_for": state.last_tick_for, "reason": request.reason}); + if let Some(cursor) = cursor { + after["cursor"] = json!(if matches!(request.change, TriggerChange::ResetCursor) { + Default::default() + } else { + cursor + }); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET status=$1, status_override=$2, last_tick_for=$3, next_due_at=$4, updated_at=$5 WHERE trigger_id=$6", + vec![Parameter::Text(status.as_str().into()), Parameter::OptionalText(state.status_override.map(|value| value.as_str().into())), Parameter::OptionalTime(state.last_tick_for), + Parameter::OptionalTime(next), Parameter::Time(now), Parameter::Id(request.trigger_id)]).await?; + Ok((transaction, (before, after))) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs new file mode 100644 index 0000000000..b258882455 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs @@ -0,0 +1,64 @@ +//! Recompute the effective schedule after code publication or an operator override. + +use chrono::{DateTime, TimeDelta, Utc}; + +use crate::declarations::models::{ + PreviousTriggerState, Schedule, TriggerDefinition, TriggerStatus, +}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::next_schedule_time::next_schedule_time; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; + +/// Replacing a future horizon must retain due ticks not yet consumed by the scheduler. +pub(crate) fn retain_calendar_ticks( + name: &str, + trigger: &mut TriggerDefinition, + state: Option<&PreviousTriggerState>, + snapshot: &SnapshotRecord, +) -> Result<(), ControlError> { + if let Some(state) = state + && let Schedule::Calendar { + times: previous_times, + .. + } = &state.definition_json.schedule + && let Schedule::Calendar { times, .. } = &mut trigger.schedule + { + times.extend( + previous_times + .iter() + .filter(|time| { + **time >= state.started_at + && **time <= snapshot.published_at + && state.last_tick_for.is_none_or(|last| **time > last) + }) + .copied(), + ); + if times.len() > MAX_TICKS_PER_BATCH { + return Err(ControlError::Invalid(format!( + "calendar {name} pending and upcoming times exceed {MAX_TICKS_PER_BATCH}; let the scheduler drain pending ticks before refreshing" + ))); + } + } + Ok(()) +} + +pub(crate) fn trigger_timing( + definition: &TriggerDefinition, + started_at: DateTime, + last_tick_for: Option>, + status_override: Option, +) -> Result<(TriggerStatus, Option>), ControlError> { + let status = status_override.unwrap_or(definition.default_status); + let after = last_tick_for.unwrap_or( + started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let next = if status == TriggerStatus::Running { + next_schedule_time(&definition.schedule, after)? + } else { + None + }; + Ok((status, next)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs new file mode 100644 index 0000000000..095353123a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/change_trigger.rs @@ -0,0 +1,77 @@ +//! Apply an operator fact and its projection in one transaction, with retry-safe request identity. + +use serde_json::json; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::declarations::_helpers::operator_state::apply_operator_change; +use crate::declarations::models::{EditableTriggerState, TriggerChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Authorization belongs to the API boundary; `actor` is the authenticated principal, not client input. +pub async fn change_trigger( + store: &ControlStore, + context: &AuditContext, + request: &TriggerChangeRequest, +) -> Result { + if request.actor.trim().is_empty() || request.actor != context.actor { + return Err(ControlError::Invalid( + "trigger changes require the authenticated audit actor".into(), + )); + } + let body = + serde_json::to_value(request).map_err(|error| ControlError::Invalid(error.to_string()))?; + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_trigger_state_events WHERE event_id=$1", + vec![Parameter::Id(request.event_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&body) { + return Err(ControlError::Invalid( + "trigger change event ID already has different content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, rows) = query( + transaction, + "SELECT t.definition_json, t.snapshot_id, s.started_at, s.last_tick_for, s.status_override \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.trigger_id=$1 AND t.removed_in_snapshot_id IS NULL", + vec![Parameter::Id(request.trigger_id)], + ) + .await?; + let state: EditableTriggerState = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("trigger does not exist in the current snapshot".into()) + })?)?; + let snapshot_id = state.snapshot_id; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state_events (event_id, trigger_id, occurred_at, snapshot_id, request_json) VALUES ($1,$2,$3,$4,$5)", + vec![Parameter::Id(request.event_id), Parameter::Id(request.trigger_id), Parameter::Time(now), Parameter::Id(state.snapshot_id), Parameter::Json(body)]).await?; + let (transaction, (before, after)) = apply_operator_change(transaction, state, request).await?; + commit( + append_audit( + transaction, + context, + (request.event_id, "trigger.changed"), + ( + json!({"trigger_id": request.trigger_id, "snapshot_id": snapshot_id}), + before, + after, + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs new file mode 100644 index 0000000000..5b04c1b88f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/frozen_definitions.rs @@ -0,0 +1,22 @@ +//! Runs resolve immutable snapshot documents, never a later deployment's job projections. + +use uuid::Uuid; + +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::models::Definitions; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn frozen_definitions( + transaction: ControlTransaction, + snapshot_id: Uuid, +) -> Result<(ControlTransaction, Definitions), ControlError> { + let (transaction, documents) = read_snapshot_documents( + transaction, + snapshot_id, + Some(&["jobs", "groups", "triggers"]), + ) + .await?; + Ok((transaction, decode_definitions(&documents)?)) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs new file mode 100644 index 0000000000..e896fba0a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -0,0 +1,92 @@ +//! Freeze job boundaries from snapshot lineage plus explicit waits, rejecting cycles. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::models::JobDefinition; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +pub fn group_dependencies( + input: &SnapshotInput, + jobs: &BTreeMap, + (members, after): (&BTreeSet, &BTreeMap>), +) -> Result>, ControlError> { + if members.is_empty() + || members.iter().any(|name| !jobs.contains_key(name)) + || after + .iter() + .any(|(name, dependencies)| !members.contains(name) || !dependencies.is_subset(members)) + { + return Err(ControlError::Invalid( + "group needs known jobs and dependencies within its membership".into(), + )); + } + let edges = lineage_dependencies(input, jobs, (members, after)); + ensure_acyclic(&edges)?; + Ok(edges) +} + +fn lineage_dependencies( + input: &SnapshotInput, + jobs: &BTreeMap, + (members, after): (&BTreeSet, &BTreeMap>), +) -> BTreeMap> { + let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for name in members { + for key in &jobs[name].selection { + owners.entry(key).or_default().insert(name); + } + } + let mut edges: BTreeMap> = BTreeMap::new(); + for name in members { + let mut dependencies = after.get(name).cloned().unwrap_or_default(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + if let Some(node) = input.nodes.get(key) + && let Some(upstream) = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + { + for upstream in upstream.iter().filter_map(serde_json::Value::as_str) { + if let Some(owners) = owners.get(upstream) { + dependencies.extend( + owners + .iter() + .filter(|owner| **owner != name) + .map(|owner| (*owner).to_owned()), + ); + } + pending.push(upstream); + } + } + } + edges.insert(name.clone(), dependencies); + } + edges +} + +fn ensure_acyclic(edges: &BTreeMap>) -> Result<(), ControlError> { + let mut remaining = edges.clone(); + while !remaining.is_empty() { + let ready: BTreeSet = remaining + .iter() + .filter(|(_, deps)| deps.is_empty()) + .map(|(name, _)| name.clone()) + .collect(); + if ready.is_empty() { + return Err(ControlError::Invalid( + "group dependencies contain a cycle".into(), + )); + } + remaining.retain(|name, deps| { + deps.retain(|dependency| !ready.contains(dependency)); + !ready.contains(name) + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs new file mode 100644 index 0000000000..10d6d3f28e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_trigger_events.rs @@ -0,0 +1,28 @@ +//! Read immutable operator changes in committed order. + +use uuid::Uuid; + +use crate::declarations::models::TriggerChangeRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_trigger_events( + store: &ControlStore, + trigger_id: Uuid, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT request_json, occurred_at, snapshot_id FROM sqb_trigger_state_events WHERE trigger_id=$1 ORDER BY occurred_at, event_id LIMIT $2", + vec![Parameter::Id(trigger_id), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let events = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(events) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs new file mode 100644 index 0000000000..c3ead93c9c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs @@ -0,0 +1,30 @@ +//! Current declarations and runtime state are read in one consistent transaction. + +use uuid::Uuid; + +use crate::declarations::models::TriggerRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_triggers( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id, t.name, t.job_key, t.definition_json, s.status, s.last_tick_for, s.next_due_at \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.location_id=$1 AND t.target=$2 AND t.removed_in_snapshot_id IS NULL ORDER BY t.name", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let triggers = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(triggers) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs new file mode 100644 index 0000000000..d87701b504 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -0,0 +1,7 @@ +pub mod change_trigger; +pub(crate) mod frozen_definitions; +pub mod group_dependencies; +pub mod list_trigger_events; +pub mod list_triggers; +pub(crate) mod publish_definitions; +pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs new file mode 100644 index 0000000000..230f3c116d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/publish_definitions.rs @@ -0,0 +1,105 @@ +//! Publish definition projections inside the current-snapshot transaction. + +use uuid::Uuid; + +use crate::declarations::_helpers::trigger_state::{retain_calendar_ticks, trigger_timing}; +use crate::declarations::models::{Definitions, PreviousTriggerState}; +use crate::sensors::main::publish_sensors::publish_sensors; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_definitions( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: Definitions, +) -> Result { + transaction = publish_jobs(transaction, snapshot, &definitions).await?; + (transaction, _) = query(transaction, + "UPDATE sqb_triggers SET removed_in_snapshot_id=$1 WHERE location_id=$2 AND target=$3 AND removed_in_snapshot_id IS NULL", + vec![Parameter::Id(snapshot.snapshot_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone())]).await?; + transaction = publish_sensors(transaction, snapshot, &definitions).await?; + for (name, mut trigger) in definitions.triggers { + let id = Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, "schedule", &name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + let previous; + (transaction, previous) = query(transaction, + "SELECT t.definition_json, s.started_at, s.last_tick_for, s.status_override FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id WHERE s.trigger_id=$1", + vec![Parameter::Id(id)]).await?; + let state = previous + .into_iter() + .next() + .map(decode_row::) + .transpose()?; + retain_calendar_ticks(&name, &mut trigger, state.as_ref(), snapshot)?; + let definition = serde_json::to_value(&trigger) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id, location_id, target, kind, name, job_key, definition_json, snapshot_id) VALUES ($1,$2,$3,'schedule',$4,$5,$6,$7) \ + ON CONFLICT (trigger_id) DO UPDATE SET job_key=excluded.job_key, definition_json=excluded.definition_json, snapshot_id=excluded.snapshot_id, removed_in_snapshot_id=NULL", + vec![Parameter::Id(id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name), + Parameter::Text(trigger.job_key.clone()), Parameter::Json(definition), Parameter::Id(snapshot.snapshot_id)]).await?; + let started_at = state.as_ref().map_or( + trigger.starts_at.unwrap_or(snapshot.published_at), + |state| state.started_at, + ); + let (status, next) = trigger_timing( + &trigger, + started_at, + state.as_ref().and_then(|state| state.last_tick_for), + state.as_ref().and_then(|state| state.status_override), + )?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id, status, started_at, next_due_at, updated_at) VALUES ($1,$2,$3,$4,$5) \ + ON CONFLICT (trigger_id) DO UPDATE SET status=excluded.status, next_due_at=excluded.next_due_at, updated_at=excluded.updated_at", + vec![Parameter::Id(id), Parameter::Text(status.as_str().into()), Parameter::Time(started_at), Parameter::OptionalTime(next), Parameter::Time(snapshot.published_at)]).await?; + } + Ok(transaction) +} + +async fn publish_jobs( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: &Definitions, +) -> Result { + for table in ["sqb_job_nodes", "sqb_jobs"] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE location_id=$1 AND target=$2"), + vec![ + Parameter::Id(snapshot.location_id), + Parameter::Text(snapshot.target.clone()), + ], + ) + .await?; + } + let jobs = definitions + .jobs + .iter() + .map(|(name, job)| (name, "job", job)) + .chain( + definitions + .groups + .iter() + .map(|(name, group)| (name, "group", &group.job)), + ); + for (name, kind, job) in jobs { + let definition = + serde_json::to_value(job).map_err(|error| ControlError::Invalid(error.to_string()))?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_jobs (location_id, target, job_key, snapshot_id, kind, definition_json, overlap_policy) VALUES ($1,$2,$3,$4,$7,$5,$6)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), + Parameter::Id(snapshot.snapshot_id), Parameter::Json(definition), Parameter::Text(job.overlap.as_str().into()), Parameter::Text(kind.into())]).await?; + for node in &job.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_job_nodes (location_id, target, job_key, node_key) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), Parameter::Text(name.clone()), Parameter::Text(node.clone())]).await?; + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs new file mode 100644 index 0000000000..0e288368d8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -0,0 +1,86 @@ +//! Validate resolved declarations before acquiring the snapshot publication lock. + +use std::collections::BTreeSet; + +use crate::declarations::_helpers::documents::decode_definitions; +use crate::declarations::main::group_dependencies::group_dependencies; +use crate::declarations::models::{Definitions, Schedule}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::models::CatchUp; +use crate::sensors::main::validate_sensors::validate_sensors; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +/// Names, references and resolved selections are checked again at the native trust boundary. +pub fn snapshot_definitions(input: &SnapshotInput) -> Result { + let definitions = decode_definitions(&input.documents)?; + for (name, job) in &definitions.jobs { + if name.trim().is_empty() + || job.description.trim().is_empty() + || job.selection.is_empty() + || job + .selection + .iter() + .any(|key| !input.nodes.contains_key(key)) + { + return Err(ControlError::Invalid(format!( + "job {name} needs a description and a nonempty selection of snapshot nodes" + ))); + } + } + for (name, group) in &definitions.groups { + let members: BTreeSet = group.jobs.keys().cloned().collect(); + let dependencies = group_dependencies(input, &definitions.jobs, (&members, &group.jobs))?; + let selection: BTreeSet = group + .jobs + .keys() + .flat_map(|name| definitions.jobs[name].selection.iter().cloned()) + .collect(); + if name.trim().is_empty() + || definitions.jobs.contains_key(name) + || group.job.description.trim().is_empty() + || group.job.selection != selection + || group.jobs != dependencies + || !group.job.settings.is_empty() + { + return Err(ControlError::Invalid(format!( + "group {name} has invalid membership, selection or dependencies" + ))); + } + } + for (name, trigger) in &definitions.triggers { + if name.trim().is_empty() + || trigger.description.trim().is_empty() + || (!definitions.jobs.contains_key(&trigger.job_key) + && !definitions.groups.contains_key(&trigger.job_key)) + { + return Err(ControlError::Invalid(format!( + "trigger {name} needs a description and a known job" + ))); + } + if let CatchUp::All { max_runs } = trigger.catch_up + && max_runs.get() > MAX_TICKS_PER_BATCH + { + return Err(ControlError::Invalid(format!( + "trigger {name} catch-up exceeds {MAX_TICKS_PER_BATCH}" + ))); + } + match &trigger.schedule { + Schedule::Cron { + expression, + timezone, + } => { + parse_cron(expression, timezone)?; + } + Schedule::Calendar { times, .. } if times.len() > MAX_TICKS_PER_BATCH => { + return Err(ControlError::Invalid(format!( + "calendar {name} exceeds {MAX_TICKS_PER_BATCH} upcoming times" + ))); + } + Schedule::Calendar { .. } => {} + } + } + validate_sensors(&definitions, input)?; + Ok(definitions) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/mod.rs b/crates/sqlbuild-orchestration/src/declarations/mod.rs new file mode 100644 index 0000000000..5359000d16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs new file mode 100644 index 0000000000..5231762051 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -0,0 +1,159 @@ +//! Resolved, serializable declarations; Python objects never enter scheduler state. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::scheduler::models::{CatchUp, Overlap}; +use crate::sensors::models::SensorDefinition; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OnFailure { + #[default] + SkipDependents, + SkipDownstreamJobs, + StopGroup, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum TriggerStatus { + #[default] + Running, + Paused, +} + +impl TriggerStatus { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Running => "running", + Self::Paused => "paused", + } + } +} + +/// Selections have already been resolved by the code location against the compiled snapshot. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JobDefinition { + pub description: String, + pub selection: BTreeSet, + #[serde(default)] + pub settings: BTreeMap, + #[serde(default)] + pub overlap: Overlap, + #[serde(default)] + pub on_failure: OnFailure, + #[serde(default)] + pub concurrency_groups: BTreeSet, +} + +/// A virtual parent shares run controls, but only its physical children claim node writers. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct GroupDefinition { + #[serde(flatten)] + pub job: JobDefinition, + pub jobs: BTreeMap>, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Schedule { + Cron { + expression: String, + timezone: String, + }, + Calendar { + times: BTreeSet>, + refresh_at: DateTime, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct TriggerDefinition { + pub description: String, + pub job_key: String, + pub schedule: Schedule, + #[serde(default)] + pub starts_at: Option>, + #[serde(default)] + pub default_status: TriggerStatus, + #[serde(default)] + pub catch_up: CatchUp, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Definitions { + pub jobs: BTreeMap, + pub groups: BTreeMap, + pub triggers: BTreeMap, + pub sensors: BTreeMap, +} + +/// Legacy schedule documents retain their stored shape; sensors add a distinct condition shape. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(untagged)] +pub enum TriggerDeclaration { + Schedule(TriggerDefinition), + Sensor(SensorDefinition), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct TriggerRecord { + pub trigger_id: uuid::Uuid, + pub name: String, + pub job_key: String, + pub definition_json: TriggerDeclaration, + pub status: TriggerStatus, + pub last_tick_for: Option>, + pub next_due_at: Option>, +} + +/// Operator changes are immutable facts; clearing an override restores the published code default. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum TriggerChange { + Enable, + Pause, + ResetDefault, + SetCursor { time: DateTime }, + ResetCursor, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct TriggerChangeRequest { + pub event_id: uuid::Uuid, + pub trigger_id: uuid::Uuid, + pub actor: String, + pub reason: String, + pub change: TriggerChange, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct TriggerChangeRecord { + pub request_json: TriggerChangeRequest, + pub occurred_at: DateTime, + pub snapshot_id: uuid::Uuid, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct EditableTriggerState { + pub(crate) definition_json: TriggerDeclaration, + pub(crate) snapshot_id: uuid::Uuid, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status_override: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct PreviousTriggerState { + pub(crate) definition_json: TriggerDefinition, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status_override: Option, +} diff --git a/crates/sqlbuild-orchestration/src/lib.rs b/crates/sqlbuild-orchestration/src/lib.rs new file mode 100644 index 0000000000..3f1421053e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/lib.rs @@ -0,0 +1,15 @@ +//! The Python-free SQLBuild control plane. + +#![forbid(unsafe_code)] + +pub mod api; +pub mod credentials; +pub mod declarations; +pub mod platform; +pub mod runs; +pub mod scheduler; +pub mod sensors; +pub mod snapshots; +pub mod store; +pub mod warehouse_metadata; +pub mod worker; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs new file mode 100644 index 0000000000..7239f2d718 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs @@ -0,0 +1,59 @@ +//! Liveness is derived across all sessions; an old instance cannot mask a healthy successor. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::platform::_helpers::observations::observe; +use crate::platform::models::{PlatformEvent, WatchdogLocation}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Locations report every 30 seconds; tolerate two missed reports. +pub(crate) async fn scan_location_page( + store: &ControlStore, + installation_id: Uuid, + after: Option<(Uuid, String)>, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let mut parameters = vec![Parameter::Time(transaction.now - TimeDelta::seconds(90))]; + let cursor = if let Some((location_id, target)) = after { + parameters.extend([Parameter::Id(location_id), Parameter::Text(target)]); + "AND (i.location_id>$2 OR (i.location_id=$2 AND i.target>$3))" + } else { + "" + }; + let (mut transaction, rows) = query( + transaction, + &format!( + "SELECT i.location_id, i.target, MAX(i.last_seen_at) AS last_seen_at \ + FROM sqb_location_instances i JOIN sqb_locations l ON l.location_id=i.location_id \ + WHERE l.archived_at IS NULL {cursor} GROUP BY i.location_id,i.target \ + HAVING MAX(i.last_seen_at)<$1 ORDER BY i.location_id,i.target LIMIT 100" + ), + parameters, + ) + .await?; + let locations: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let next = locations + .last() + .map(|row| (row.location_id, row.target.clone())); + for location in locations { + transaction = observe( + transaction, + installation_id, + PlatformEvent::LocationBroken { + location_id: location.location_id, + target: location.target, + last_seen_at: location.last_seen_at, + }, + ) + .await?; + } + commit(transaction).await?; + Ok(next) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs new file mode 100644 index 0000000000..a13e3fabb1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/location_heartbeat.rs @@ -0,0 +1,50 @@ +//! A heartbeat replaces one instance's liveness, never appending an execution fact. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::models::LocationHealthReport; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn location_heartbeat( + transaction: ControlTransaction, + location_id: Uuid, + report: &LocationHealthReport, +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT instance.location_id AS instance_location, instance.health_json AS report \ + FROM sqb_locations location LEFT JOIN sqb_location_instances instance ON instance.instance_id=$1 \ + WHERE location.location_id=$2 AND location.archived_at IS NULL", + vec![Parameter::Id(report.session_id), Parameter::Id(location_id)]).await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("active location is required".into()))?; + if let Some(previous) = row.get("report").filter(|value| !value.is_null()) { + let previous: LocationHealthReport = serde_json::from_value(previous.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if row.get("instance_location") != Some(&json!(location_id)) + || previous.target != report.target + { + return Err(ControlError::Invalid( + "location instance belongs to another scope".into(), + )); + } + if previous == *report { + return Ok((transaction, false)); + } + if previous.event_id == report.event_id { + return Err(ControlError::Invalid( + "location health event ID already has different content".into(), + )); + } + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_instances (instance_id, location_id, target, connected_at, last_seen_at, health_json) \ + VALUES ($1, $2, $3, $4, $4, $5) ON CONFLICT (instance_id) DO UPDATE SET last_seen_at=$4, health_json=$5", + vec![Parameter::Id(report.session_id), Parameter::Id(location_id), Parameter::Text(report.target.clone()), + Parameter::Time(now), Parameter::Json(json!(report))]).await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs new file mode 100644 index 0000000000..fddc0b0e5a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod liveness; +pub(crate) mod location_heartbeat; +pub(crate) mod observations; +pub(crate) mod watchdog; +pub(crate) mod webhook_delivery; +pub(crate) mod work_client; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs new file mode 100644 index 0000000000..8f5779d7fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/observations.rs @@ -0,0 +1,57 @@ +//! Replicas share a deterministic observation key and a serialized append transaction. + +use chrono::{DurationRound, TimeDelta}; +use serde_json::json; +use uuid::Uuid; + +use crate::platform::models::PlatformEvent; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +pub(crate) async fn observe( + transaction: ControlTransaction, + installation_id: Uuid, + event: PlatformEvent, +) -> Result { + let occurred_at = transaction + .now + .duration_trunc(TimeDelta::seconds(30)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let payload = json!({"platform": event}); + let identity = serde_json::to_vec(&(occurred_at, &event)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let event_id = Uuid::new_v5(&installation_id, &identity); + let (transaction, previous) = query( + transaction, + "SELECT 1 AS present FROM recall_events WHERE event_id=$1 AND occurred_at=$2 LIMIT 1", + vec![ + Parameter::Text(event_id.to_string()), + Parameter::Time(occurred_at), + ], + ) + .await?; + if !previous.is_empty() { + return Ok(transaction); + } + let (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("platform/{installation_id}/{}", occurred_at.date_naive()), + event_type: event.event_type(), + producer: "platform", + occurred_at, + run_id: None, + payload, + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "platform observation conflicts with a durable fact".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs new file mode 100644 index 0000000000..89ca089b53 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -0,0 +1,132 @@ +//! Health scanning releases the writer lock between location and trigger pages. + +use chrono::TimeDelta; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::declarations::models::Schedule; +use crate::platform::_helpers::liveness::scan_location_page; +use crate::platform::_helpers::observations::observe; +use crate::platform::models::{InstallationHealth, PlatformEvent, WatchdogTrigger}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn scan_health( + store: &ControlStore, + replica_id: Uuid, + shutdown: &watch::Receiver, +) -> Result<(), ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT i.installation_id, CASE WHEN l.expires_at>$1 THEN 1 ELSE 0 END AS leader \ + FROM sqb_installation i LEFT JOIN sqb_scheduler_lease l ON l.name='scheduler' WHERE i.singleton=1", + vec![Parameter::Time(now)]).await?; + let health: InstallationHealth = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("control store has no installation".into()))?, + )?; + let mut transaction = transaction; + if health.leader == 0 { + transaction = observe( + transaction, + health.installation_id, + PlatformEvent::SchedulerNoLeader, + ) + .await?; + } + commit(transaction).await?; + let mut location_after = None; + while !*shutdown.borrow() { + location_after = scan_location_page(store, health.installation_id, location_after).await?; + if location_after.is_none() { + break; + } + tokio::task::yield_now().await; + } + let mut after = None; + while !*shutdown.borrow() { + after = scan_trigger_page(store, health.installation_id, after).await?; + if after.is_none() { + break; + } + tokio::task::yield_now().await; + } + if *shutdown.borrow() { + return Ok(()); + } + let transaction = begin(store, TransactionMode::Write).await?; + commit( + observe( + transaction, + health.installation_id, + PlatformEvent::Heartbeat { replica_id }, + ) + .await?, + ) + .await?; + Ok(()) +} + +async fn scan_trigger_page( + store: &ControlStore, + installation_id: Uuid, + after: Option, +) -> Result, ControlError> { + let cursor = after.map_or("", |_| "AND t.trigger_id>$1"); + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (mut transaction, rows) = query( + transaction, + &format!( + "SELECT t.trigger_id, t.definition_json, s.next_due_at \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE t.kind='schedule' AND s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + {cursor} ORDER BY t.trigger_id LIMIT 100" + ), + after.into_iter().map(Parameter::Id).collect(), + ) + .await?; + let triggers: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let next = triggers.last().map(|trigger| trigger.trigger_id); + for trigger in triggers { + if let Some(scheduled_for) = trigger.next_due_at + && scheduled_for < now - TimeDelta::seconds(30) + { + transaction = observe( + transaction, + installation_id, + PlatformEvent::TickOverdue { + trigger_id: trigger.trigger_id, + scheduled_for, + }, + ) + .await?; + } + if let Schedule::Calendar { times, refresh_at } = trigger.definition_json.schedule + && times + .last() + .is_none_or(|last| *last <= now + TimeDelta::days(1)) + { + transaction = observe( + transaction, + installation_id, + PlatformEvent::CalendarHorizonLow { + trigger_id: trigger.trigger_id, + refresh_at, + horizon_end: times.last().copied(), + }, + ) + .await?; + } + } + commit(transaction).await?; + Ok(next) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs new file mode 100644 index 0000000000..eb5547281b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/webhook_delivery.rs @@ -0,0 +1,55 @@ +//! Ordered, at-least-once delivery; a response is acknowledged only after HTTP success. + +use crate::platform::main::read_platform_events::read_platform_events; +use crate::platform::models::PlatformWebhook; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// A lost acknowledgement may resend the same event_id; receivers must deduplicate by that ID. +pub(crate) async fn deliver_platform_events( + store: &ControlStore, + sink: &PlatformWebhook, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT COALESCE(MAX(storage_id), 0) AS cursor FROM sqb_platform_sink_deliveries WHERE sink_id=$1", + vec![Parameter::Text(sink.sink_id.clone())], + ) + .await?; + let cursor = serde_json::from_value(rows[0]["cursor"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, events) = read_platform_events(transaction, cursor).await?; + commit(transaction).await?; + let count = events.len(); + for event in events { + let response = sink + .client + .post(sink.url.clone()) + .header("Idempotency-Key", event.event_id.to_string()) + .json(&event) + .send() + .await + .map_err(|_| { + ControlError::Invalid("platform webhook delivery failed; event retained".into()) + })?; + if !response.status().is_success() { + return Err(ControlError::Invalid(format!( + "platform webhook returned HTTP {}; event retained", + response.status().as_u16() + ))); + } + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_platform_sink_deliveries (sink_id, storage_id, event_id, delivered_at) \ + VALUES ($1,$2,$3,$4) ON CONFLICT (sink_id, storage_id) DO NOTHING", + vec![Parameter::Text(sink.sink_id.clone()), Parameter::Integer(event.storage_id), + Parameter::Text(event.event_id.to_string()), Parameter::Time(now)]).await?; + commit(transaction).await?; + } + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs new file mode 100644 index 0000000000..5c1351ac68 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/work_client.rs @@ -0,0 +1,72 @@ +//! The authenticated client verifies every leased request's location, target and instance. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::models::MutationResponse; +use crate::platform::models::{ + LocationWorkClaim, LocationWorkClient, LocationWorkPoll, LocationWorkResult, +}; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +impl LocationWorkClient { + /// Metadata shares the location's scoped pool, not the scheduler's execution authority. + pub async fn sync_source_freshness( + &self, + target: &str, + batch: &SourceFreshnessBatch, + ) -> Result { + let endpoint = self.connection.endpoint(&[ + "api", + "locations", + &self.location_id.to_string(), + "targets", + target, + "metadata", + "source-freshness", + ])?; + let response: MutationResponse = + client_response(self.connection.client.post(endpoint).json(batch)).await?; + Ok(response.applied) + } + + pub async fn poll( + &self, + poll: &LocationWorkPoll, + ) -> Result, ClientError> { + let claim: Option = client_response( + self.connection + .client + .post(format!("{}poll", self.work_url)) + .json(poll), + ) + .await?; + if claim.as_ref().is_some_and(|claim| { + claim.request.location_id != self.location_id + || claim.request.target != poll.target + || claim.session_id != poll.session_id + }) { + return Err(ClientError::Configuration( + "work response does not match this location, target and instance", + )); + } + Ok(claim) + } + + /// Retain the same request, claim and result when retrying an uncertain acknowledgement. + pub async fn complete( + &self, + request_id: Uuid, + result: &LocationWorkResult, + ) -> Result { + let response: MutationResponse = client_response( + self.connection + .client + .post(format!("{}{request_id}", self.work_url)) + .json(result), + ) + .await?; + Ok(response.applied) + } +} diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs new file mode 100644 index 0000000000..75d8a07df7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -0,0 +1,8 @@ +//! Bounds shared by the platform event reader and webhook dispatcher. + +/// Operator-owned delivery endpoint; never inherited by project workers or printed in logs. +pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL"; + +pub(crate) const PLATFORM_EVENT_PAGE_SIZE: i64 = 100; +pub(crate) const SINK_REQUEST_TIMEOUT_SECONDS: u64 = 10; +pub(crate) const SINK_POLL_SECONDS: u64 = 5; diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs new file mode 100644 index 0000000000..8d071b1626 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -0,0 +1,72 @@ +//! Serialized leases fence results from replaced location instances and uncertain poll retries. + +use chrono::Duration; +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::require_instance::require_instance; +use crate::platform::models::{LocationWorkClaim, LocationWorkPoll}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn claim_location_work( + store: &ControlStore, + location_id: Uuid, + poll: &LocationWorkPoll, +) -> Result, ControlError> { + let transaction = require_instance( + begin(store, TransactionMode::Write).await?, + (location_id, poll.session_id, &poll.target), + ) + .await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT w.request_id FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ + AND w.expires_at>$3 AND NOT EXISTS (SELECT 1 FROM sqb_location_work_results r WHERE r.request_id=w.request_id) \ + AND (w.claimed_until IS NULL OR w.claimed_until<=$3 OR w.session_id=$4) \ + AND (w.request_json->'work'->>'kind'='check_connection' OR EXISTS ( \ + SELECT 1 FROM sqb_sensor_evaluation_events e JOIN sqb_triggers t ON t.trigger_id=e.trigger_id \ + JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id JOIN sqb_scheduler_lease l ON l.name='scheduler' \ + WHERE e.evaluation_id=w.request_id AND e.phase='requested' AND s.status='running' \ + AND t.snapshot_id=w.snapshot_id AND t.removed_in_snapshot_id IS NULL AND l.expires_at>$3 \ + AND l.epoch=CAST(e.fact_json->'request'->>'epoch' AS BIGINT) \ + AND NOT EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events f \ + WHERE f.evaluation_id=e.evaluation_id AND f.phase<>'requested'))) \ + ORDER BY w.requested_at, w.request_id LIMIT 1", + vec![Parameter::Id(location_id), Parameter::Text(poll.target.clone()), Parameter::Time(now), Parameter::Id(poll.session_id)]).await?; + let Some(row) = rows.first() else { + commit(transaction).await?; + return Ok(None); + }; + let request_id = serde_json::from_value(row["request_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid work identity: {error}")))?; + let (transaction, stored) = read_location_work(transaction, request_id).await?; + let stored = stored.ok_or_else(|| ControlError::Invalid("location work disappeared".into()))?; + let retained = stored.session_id == Some(poll.session_id) + && stored.claimed_until.is_some_and(|until| until > now); + let (claim_id, lease_expires_at) = if retained { + ( + stored.claim_id.ok_or(ControlError::Fenced)?, + stored.claimed_until.ok_or(ControlError::Fenced)?, + ) + } else { + ( + Uuid::now_v7(), + (now + Duration::seconds(60)).min(stored.record.expires_at), + ) + }; + let (transaction, _) = query(transaction, + "UPDATE sqb_location_work SET claim_id=$1, session_id=$2, claimed_until=$3 WHERE request_id=$4", + vec![Parameter::Id(claim_id), Parameter::Id(poll.session_id), Parameter::Time(lease_expires_at), Parameter::Id(request_id)]).await?; + commit(transaction).await?; + Ok(Some(LocationWorkClaim { + request: stored.record.request_json, + generation: stored.record.generation, + claim_id, + session_id: poll.session_id, + lease_expires_at, + })) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs new file mode 100644 index 0000000000..cc10fab99c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -0,0 +1,81 @@ +//! A completion is immutable; only the current unexpired claim may publish its result. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::require_instance::require_instance; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkResult}; +use crate::sensors::constants::MAX_SENSOR_RUN_REQUESTS; +use crate::sensors::models::CustomSensorOutcome; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn complete_location_work( + store: &ControlStore, + (location_id, request_id): (Uuid, Uuid), + result: &LocationWorkResult, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, stored) = read_location_work(transaction, request_id).await?; + let stored = stored + .filter(|stored| stored.record.request_json.location_id == location_id) + .ok_or_else(|| ControlError::Invalid("work is not assigned to this location".into()))?; + let transaction = require_instance( + transaction, + ( + location_id, + result.session_id, + &stored.record.request_json.target, + ), + ) + .await?; + if let Some(previous) = stored.record.result_json { + if previous != *result { + return Err(ControlError::Invalid( + "location work already has a different result".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + if stored.claim_id != Some(result.claim_id) + || stored.session_id != Some(result.session_id) + || stored + .claimed_until + .is_none_or(|until| until <= transaction.now) + || stored.record.expires_at <= transaction.now + { + return Err(ControlError::Fenced); + } + let valid = match (&stored.record.request_json.work, &result.outcome) { + (_, LocationWorkOutcome::Superseded {}) + | (LocationWork::CheckConnection {}, LocationWorkOutcome::Connection { .. }) => true, + ( + LocationWork::EvaluateSensor { .. }, + LocationWorkOutcome::SensorEvaluation { outcome }, + ) => match outcome { + CustomSensorOutcome::Completed { run_keys, .. } => { + run_keys.len() <= MAX_SENSOR_RUN_REQUESTS + && run_keys.iter().all(|key| !key.trim().is_empty()) + } + CustomSensorOutcome::Skipped { reason, .. } => !reason.trim().is_empty(), + CustomSensorOutcome::Failed | CustomSensorOutcome::TimedOut => true, + }, + _ => false, + }; + if !valid { + return Err(ControlError::Invalid( + "location result does not match its work".into(), + )); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work_results (request_id,result_json,completed_at) VALUES ($1,$2,$3)", + vec![Parameter::Id(request_id), Parameter::Json(json!(result)), Parameter::Time(now)]).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs new file mode 100644 index 0000000000..a0df47b88d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/connect_location_work.rs @@ -0,0 +1,21 @@ +//! Reuse one credential-bearing pool for the location's outbound work channel. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::connect_client::connect_client; +use crate::platform::models::LocationWorkClient; + +pub fn connect_location_work( + (server_url, bearer): (&str, &str), + location_id: Uuid, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let work_url = + connection.endpoint(&["api", "locations", &location_id.to_string(), "work", ""])?; + Ok(LocationWorkClient { + connection, + work_url, + location_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs b/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs new file mode 100644 index 0000000000..4a715b9ec8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/connect_platform_webhook.rs @@ -0,0 +1,31 @@ +//! Validate operator-owned delivery configuration without exposing credentials in errors. + +use std::time::Duration; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::platform::constants::SINK_REQUEST_TIMEOUT_SECONDS; +use crate::platform::models::PlatformWebhook; +use crate::store::errors::ControlError; + +/// Changing the endpoint creates a new consumer, starting from the retained event history. +pub fn connect_platform_webhook(endpoint: &str) -> Result { + let url = reqwest::Url::parse(endpoint) + .map_err(|_| ControlError::Invalid("platform webhook needs an HTTP(S) URL".into()))?; + if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { + return Err(ControlError::Invalid( + "platform webhook needs an HTTP(S) URL".into(), + )); + } + let sink_id = sha256_hex(url.as_str().as_bytes()); + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(SINK_REQUEST_TIMEOUT_SECONDS)) + .build() + .map_err(|_| ControlError::Invalid("platform webhook client could not start".into()))?; + Ok(PlatformWebhook { + client, + url, + sink_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs new file mode 100644 index 0000000000..977a61e6c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/location_health.rs @@ -0,0 +1,63 @@ +//! Read bounded recent observations and instance liveness; consumers derive freshness. + +use uuid::Uuid; + +use crate::platform::models::LocationHealthObservation; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn location_health( + store: &ControlStore, + location_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let today = transaction.now.date_naive(); + let yesterday = today - chrono::TimeDelta::days(1); + let (transaction, rows) = query( + transaction, + "SELECT received_at, payload FROM (\ + SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ + WHERE invocation_id=$1 AND event_type='location_health' ORDER BY invocation_sequence DESC LIMIT 100) current_day \ + UNION ALL \ + SELECT * FROM (SELECT occurred_at AS received_at, invocation_sequence, payload FROM recall_events \ + WHERE invocation_id=$2 AND event_type='location_health' ORDER BY invocation_sequence DESC LIMIT 100) previous_day\ + ) observations ORDER BY received_at DESC, invocation_sequence DESC LIMIT 100", + vec![ + Parameter::Text(format!("location/{location_id}/{today}")), + Parameter::Text(format!("location/{location_id}/{yesterday}")), + ], + ) + .await?; + let mut observations: Vec = rows + .into_iter() + .map(|mut row| { + let report = row + .remove("payload") + .and_then(|mut payload| payload.get_mut("report").map(serde_json::Value::take)) + .ok_or_else(|| ControlError::Invalid("location health report is missing".into()))?; + row.insert("report".into(), report); + decode_row(row) + }) + .collect::>()?; + let (transaction, instances) = query( + transaction, + "SELECT last_seen_at AS received_at, health_json AS report FROM sqb_location_instances \ + WHERE location_id=$1 ORDER BY last_seen_at DESC LIMIT 100", + vec![Parameter::Id(location_id)], + ) + .await?; + observations.extend( + instances + .into_iter() + .map(decode_row) + .collect::, _>>()?, + ); + observations.sort_by_key(|observation| std::cmp::Reverse(observation.received_at)); + observations.truncate(100); + commit(transaction).await?; + Ok(observations) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs new file mode 100644 index 0000000000..8a8783c56f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/location_work.rs @@ -0,0 +1,20 @@ +//! Read one request and its durable result without persisting a derived pending/expired status. + +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::LocationWorkRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn location_work( + store: &ControlStore, + request_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, record) = read_location_work(transaction, request_id).await?; + commit(transaction).await?; + Ok(record.map(|stored| stored.record)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs new file mode 100644 index 0000000000..8124f49518 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -0,0 +1,14 @@ +pub mod claim_location_work; +pub mod complete_location_work; +pub mod connect_location_work; +pub mod connect_platform_webhook; +pub mod location_health; +pub mod location_work; +pub(crate) mod read_location_work; +pub(crate) mod read_platform_events; +pub mod report_location_health; +pub mod report_remote_location_health; +pub mod request_location_work; +pub(crate) mod require_instance; +pub mod run_platform_sink; +pub mod run_watchdog; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs new file mode 100644 index 0000000000..8401a196a0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs @@ -0,0 +1,24 @@ +//! Read durable work inside a caller's existing transaction, including fenced sensor completion. + +use uuid::Uuid; + +use crate::platform::models::StoredLocationWork; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_location_work( + transaction: ControlTransaction, + request_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT w.request_json, w.requested_by, w.requested_at, w.expires_at, w.generation, \ + w.claim_id, w.session_id, w.claimed_until, r.result_json, r.completed_at \ + FROM sqb_location_work w LEFT JOIN sqb_location_work_results r ON r.request_id=w.request_id \ + WHERE w.request_id=$1", vec![Parameter::Id(request_id)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs b/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs new file mode 100644 index 0000000000..48a38850e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_platform_events.rs @@ -0,0 +1,22 @@ +//! One authoritative namespace filter protects every platform stream consumer from worker input. + +use crate::platform::constants::PLATFORM_EVENT_PAGE_SIZE; +use crate::platform::models::PlatformEventEnvelope; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_platform_events( + transaction: ControlTransaction, + after: i64, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT storage_id,event_id,occurred_at,payload FROM recall_events WHERE producer='platform' \ + AND invocation_id LIKE 'platform/%' AND storage_id>$1 ORDER BY storage_id LIMIT $2", + vec![Parameter::Integer(after), Parameter::Integer(PLATFORM_EVENT_PAGE_SIZE)]).await?; + Ok(( + transaction, + rows.into_iter().map(decode_row).collect::>()?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs new file mode 100644 index 0000000000..389d80fa18 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs @@ -0,0 +1,104 @@ +//! Append scoped location observations without turning health into mutable lifecycle state. + +use serde_json::json; +use uuid::Uuid; + +use crate::platform::_helpers::location_heartbeat::location_heartbeat; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlStore, Parameter, TransactionMode}; + +pub async fn report_location_health( + store: &ControlStore, + location_id: Uuid, + report: &LocationHealthReport, +) -> Result { + if report.target.trim().is_empty() { + return Err(ControlError::Invalid( + "location health target is invalid".into(), + )); + } + let generation = match report.health { + LocationHealth::Heartbeat {} => None, + LocationHealth::CompileFailed { generation } + | LocationHealth::Connection { generation, .. } => Some(generation), + }; + if generation.is_some_and(|generation| generation < 1) { + return Err(ControlError::Invalid( + "location health generation must be positive".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + if generation.is_none() { + let (transaction, applied) = location_heartbeat(transaction, location_id, report).await?; + commit(transaction).await?; + return Ok(applied); + } + let invocation_id = format!("location/{location_id}/{}", transaction.now.date_naive()); + let event_id = Uuid::new_v5(&location_id, report.event_id.as_bytes()); + let payload = json!({"location_id": location_id, "report": report}); + let (transaction, previous) = query( + transaction, + "SELECT payload FROM recall_events WHERE event_id=$1 LIMIT 1", + vec![Parameter::Text(event_id.to_string())], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("payload") != Some(&payload) { + return Err(ControlError::Invalid( + "location health event ID already has different content".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, location) = query( + transaction, + "SELECT location_id FROM sqb_locations WHERE location_id=$1 AND archived_at IS NULL", + vec![Parameter::Id(location_id)], + ) + .await?; + if location.is_empty() { + return Err(ControlError::Invalid("active location is required".into())); + } + let transaction = if let LocationHealth::Connection { generation, .. } = report.health { + let (transaction, snapshot) = query( + transaction, + "SELECT snapshot_id FROM sqb_snapshots WHERE location_id=$1 AND target=$2 AND generation=$3", + vec![Parameter::Id(location_id), Parameter::Text(report.target.clone()), Parameter::Integer(generation)], + ).await?; + if snapshot.is_empty() { + return Err(ControlError::Invalid( + "connection health requires a published generation".into(), + )); + } + transaction + } else { + transaction + }; + let occurred_at = transaction.now; + let (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id, + event_type: "location_health", + producer: "location", + occurred_at, + run_id: None, + payload, + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "location health conflicts with a durable fact".into(), + )); + } + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs new file mode 100644 index 0000000000..5963cf20b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/report_remote_location_health.rs @@ -0,0 +1,21 @@ +//! The code location has only its scoped HTTP credential, never a store handle. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::api::models::MutationResponse; +use crate::platform::models::LocationHealthReport; + +pub async fn report_remote_location_health( + (server_url, bearer): (&str, &str), + location_id: Uuid, + report: &LocationHealthReport, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&["api", "locations", &location_id.to_string(), "health"])?; + let response: MutationResponse = + client_response(connection.client.post(url).json(report)).await?; + Ok(response.applied) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs new file mode 100644 index 0000000000..df0c73f23d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -0,0 +1,86 @@ +//! Only trusted store owners enqueue named work; retries preserve the original generation/deadline. + +use chrono::Duration; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn request_location_work( + store: &ControlStore, + context: &AuditContext, + request: &LocationWorkRequest, +) -> Result { + if matches!(request.work, LocationWork::EvaluateSensor { .. }) { + return Err(ControlError::Invalid( + "sensor work must originate from a fenced evaluation".into(), + )); + } + if request.target.trim().is_empty() + || request.reason.trim().is_empty() + || context.actor.trim().is_empty() + { + return Err(ControlError::Invalid( + "location work needs a target, reason and trusted actor".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = read_location_work(transaction, request.request_id).await?; + if let Some(previous) = previous { + if previous.record.request_json != *request || previous.record.requested_by != context.actor + { + return Err(ControlError::Invalid( + "location work ID already has different content or actor".into(), + )); + } + commit(transaction).await?; + return Ok(false); + } + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.generation FROM sqb_current_snapshots c \ + JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id JOIN sqb_locations l ON l.location_id=c.location_id \ + WHERE c.location_id=$1 AND c.target=$2 AND l.archived_at IS NULL", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone())]).await?; + let snapshot = rows + .first() + .ok_or_else(|| ControlError::Invalid("location work requires a published target".into()))?; + let snapshot_id = serde_json::from_value(snapshot["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid snapshot identity: {error}")))?; + let generation = snapshot["generation"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("invalid snapshot generation".into()))?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Id(snapshot_id), Parameter::Integer(generation), Parameter::Json(json!(request)), + Parameter::Text(context.actor.clone()), Parameter::Time(now), Parameter::Time(now + Duration::minutes(5))]).await?; + commit( + append_audit( + transaction, + context, + ( + Uuid::new_v5(&request.request_id, b"location-work-requested"), + "location.work_requested", + ), + ( + json!({"location_id": request.location_id, "target": request.target}), + json!(null), + json!(request), + ), + ) + .await?, + ) + .await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs b/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs new file mode 100644 index 0000000000..9d64b6068f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/require_instance.rs @@ -0,0 +1,23 @@ +//! Named work and warehouse observations share the registered location/target boundary. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn require_instance( + transaction: ControlTransaction, + (location_id, session_id, target): (Uuid, Uuid, &str), +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT i.instance_id FROM sqb_location_instances i JOIN sqb_locations l ON l.location_id=i.location_id \ + WHERE i.instance_id=$1 AND i.location_id=$2 AND i.target=$3 AND l.archived_at IS NULL", + vec![Parameter::Id(session_id), Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "a registered instance of this location and target is required".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs b/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs new file mode 100644 index 0000000000..02757c1627 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/run_platform_sink.rs @@ -0,0 +1,48 @@ +//! Deliver platform health independently of user interpreters and scheduler leadership. + +use std::time::Duration; + +use tokio::sync::watch; + +use crate::platform::_helpers::webhook_delivery::deliver_platform_events; +use crate::platform::constants::SINK_POLL_SECONDS; +use crate::platform::models::PlatformWebhook; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub async fn run_platform_sink( + store: &ControlStore, + sink: PlatformWebhook, + mut shutdown: watch::Receiver, +) -> Result<(), ControlError> { + let mut failed = false; + while !*shutdown.borrow() { + tokio::select! { + result = deliver_platform_events(store, &sink) => { + match result { + Ok(count) => { + if failed { + eprintln!("Platform webhook delivery recovered"); + } + failed = false; + if count > 0 { continue; } + }, + Err(_) => { + if !failed { + eprintln!("Platform webhook delivery failed; retaining events and retrying"); + } + failed = true; + }, + } + }, + result = shutdown.changed() => { + if result.is_err() || *shutdown.borrow() { break; } + }, + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(SINK_POLL_SECONDS)) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs b/crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs new file mode 100644 index 0000000000..d8a72dc0fb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/run_watchdog.rs @@ -0,0 +1,25 @@ +//! Every server observes health independently; scheduler leadership is not required. + +use std::time::Duration; + +use tokio::sync::watch; +use uuid::Uuid; + +use crate::platform::_helpers::watchdog::scan_health; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub async fn run_watchdog( + store: &ControlStore, + replica_id: Uuid, + mut shutdown: watch::Receiver, +) -> Result<(), ControlError> { + while !*shutdown.borrow() { + scan_health(store, replica_id, &shutdown).await?; + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(30)) => {}, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/platform/mod.rs b/crates/sqlbuild-orchestration/src/platform/mod.rs new file mode 100644 index 0000000000..d58521e0f6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/platform/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs new file mode 100644 index 0000000000..fab9c20eab --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -0,0 +1,236 @@ +//! Platform health is a typed stream of observations, independent of user code. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::api::models::ControlClient; +use crate::declarations::models::TriggerDefinition; +use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; + +/// One pooled outbound connection; the location never receives a store handle. +#[derive(Clone, Debug)] +pub struct LocationWorkClient { + pub(crate) connection: ControlClient, + pub(crate) work_url: reqwest::Url, + pub(crate) location_id: Uuid, +} + +/// Locations report bounded facts, never connection strings or user exception text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationHealth { + Heartbeat {}, + CompileFailed { generation: i64 }, + Connection { generation: i64, healthy: bool }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationHealthReport { + pub event_id: Uuid, + pub session_id: Uuid, + pub target: String, + pub health: LocationHealth, +} + +/// Receipt time belongs to the server; consumers derive freshness rather than storing availability. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LocationHealthObservation { + pub received_at: DateTime, + pub report: LocationHealthReport, +} + +/// Only named location operations cross this boundary, never arbitrary SQL or Python. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationWork { + CheckConnection {}, + EvaluateSensor { + #[schema(value_type = Object)] + evaluation: Box, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkRequest { + pub request_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub reason: String, + pub work: LocationWork, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkPoll { + pub session_id: Uuid, + pub target: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkClaim { + pub request: LocationWorkRequest, + pub generation: i64, + pub claim_id: Uuid, + pub session_id: Uuid, + pub lease_expires_at: DateTime, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LocationWorkOutcome { + Connection { healthy: bool }, + SensorEvaluation { outcome: CustomSensorOutcome }, + Superseded {}, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct LocationWorkResult { + pub claim_id: Uuid, + pub session_id: Uuid, + pub outcome: LocationWorkOutcome, +} + +/// Expiry is derived from the deadline; only requests, leases and completed results are stored. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LocationWorkRecord { + pub request_json: LocationWorkRequest, + pub requested_by: String, + pub requested_at: DateTime, + pub expires_at: DateTime, + pub generation: i64, + pub result_json: Option, + pub completed_at: Option>, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct StoredLocationWork { + #[serde(flatten)] + pub(crate) record: LocationWorkRecord, + pub(crate) claim_id: Option, + pub(crate) session_id: Option, + pub(crate) claimed_until: Option>, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum PlatformEventKind { + Heartbeat, + SchedulerNoLeader, + TickOverdue, + CalendarHorizonLow, + LocationBroken, + LaunchFailed, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PlatformEvent { + Heartbeat { + replica_id: Uuid, + }, + SchedulerNoLeader, + TickOverdue { + trigger_id: Uuid, + scheduled_for: DateTime, + }, + CalendarHorizonLow { + trigger_id: Uuid, + refresh_at: DateTime, + horizon_end: Option>, + }, + /// All observed instances of this location/target have missed three heartbeats. + LocationBroken { + location_id: Uuid, + target: String, + last_seen_at: DateTime, + }, + /// A launch attempt or its response failed; the worker may still start or finish. + LaunchFailed { + run_id: Uuid, + claim_id: Uuid, + code: String, + }, +} + +impl PlatformEvent { + #[must_use] + pub const fn kind(&self) -> PlatformEventKind { + match self { + Self::Heartbeat { .. } => PlatformEventKind::Heartbeat, + Self::SchedulerNoLeader => PlatformEventKind::SchedulerNoLeader, + Self::TickOverdue { .. } => PlatformEventKind::TickOverdue, + Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, + Self::LocationBroken { .. } => PlatformEventKind::LocationBroken, + Self::LaunchFailed { .. } => PlatformEventKind::LaunchFailed, + } + } + + #[must_use] + pub const fn event_type(&self) -> &'static str { + match self { + Self::Heartbeat { .. } => "platform_heartbeat", + Self::SchedulerNoLeader => "platform_scheduler_no_leader", + Self::TickOverdue { .. } => "platform_tick_overdue", + Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", + Self::LocationBroken { .. } => "platform_location_broken", + Self::LaunchFailed { .. } => "platform_launch_failed", + } + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct InstallationHealth { + pub(crate) installation_id: Uuid, + pub(crate) leader: i64, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct WatchdogTrigger { + pub(crate) trigger_id: Uuid, + pub(crate) definition_json: TriggerDefinition, + pub(crate) next_due_at: Option>, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct WatchdogLocation { + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) last_seen_at: DateTime, +} + +/// The endpoint may contain credentials; neither errors nor Debug reveal it. +#[derive(Clone)] +pub struct PlatformWebhook { + pub(crate) client: reqwest::Client, + pub(crate) url: reqwest::Url, + pub(crate) sink_id: String, +} + +impl std::fmt::Debug for PlatformWebhook { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PlatformWebhook") + .finish_non_exhaustive() + } +} + +/// Stable event identity permits receiver-side deduplication after uncertain acknowledgements. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlatformEventEnvelope { + pub storage_id: i64, + pub event_id: Uuid, + pub occurred_at: DateTime, + pub payload: PlatformEventPayload, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlatformEventPayload { + pub platform: PlatformEvent, +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs new file mode 100644 index 0000000000..f893687365 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs @@ -0,0 +1,186 @@ +//! Reconstruct global parent allocations and remaining child demands in the reservation transaction. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::capacity::{claim_resources, parent_resources}; +use crate::runs::_helpers::groups::selection::child_ready; +use crate::runs::models::{ + ConcurrencyResource, GroupCapacity, GroupRun, QueueCandidate, RunKind, RunStatus, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::ControlTransaction; + +pub(crate) async fn group_capacity( + transaction: ControlTransaction, + limits: &BTreeMap, + eligible: Option<&BTreeSet<(Uuid, String)>>, +) -> Result<(ControlTransaction, GroupCapacity), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE (r.kind='group' AND r.status IN ('queued','running')) \ + OR (r.status='queued' AND r.cancel_requested_at IS NULL AND l.run_id IS NULL) \ + ORDER BY r.priority DESC, r.queued_at, r.run_id", vec![]).await?; + let runs: Vec = rows.into_iter().map(decode_row).collect::>()?; + let parents: BTreeMap<_, _> = runs + .iter() + .filter(|run| run.request_json.kind == RunKind::Group) + .map(|run| (run.request_json.run_id, run)) + .collect(); + let (transaction, slots) = query(transaction, + "SELECT s.group_key, s.run_id FROM sqb_concurrency_slots s JOIN sqb_runs r ON r.run_id=s.run_id WHERE r.kind='group' AND r.status='running'", vec![]).await?; + let mut capacity = GroupCapacity { + available: limits + .iter() + .map(|(key, limit)| { + ( + ConcurrencyResource::Limit(key.clone()), + i64::from(limit.get()), + ) + }) + .collect(), + held: BTreeMap::new(), + needed: parents.keys().map(|id| (*id, BTreeSet::new())).collect(), + waiting: runs + .iter() + .filter(|run| { + run.state.status == RunStatus::Queued + && run.state.cancel_requested_at.is_none() + && eligible.is_none_or(|scopes| { + scopes.contains(&( + run.request_json.location_id, + run.request_json.target.clone(), + )) + }) + && run.request_json.parent_run_id.is_none_or(|id| { + parents.get(&id).is_some_and(|parent| { + parent.state.status == RunStatus::Running + && parent.state.cancel_requested_at.is_none() + }) + }) + }) + .map(|run| QueueCandidate { + request_json: run.request_json.clone(), + queued_at: run.queued_at, + }) + .collect(), + }; + for (id, parent) in &parents { + if parent.state.status == RunStatus::Running { + let held = capacity.held.entry(*id).or_default(); + held.extend( + parent_resources(&parent.request_json) + .into_iter() + .filter(|resource| matches!(resource, ConcurrencyResource::Overlap(..))), + ); + } + } + for slot in slots { + let id = serde_json::from_value(slot["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let key = slot["group_key"] + .as_str() + .ok_or_else(|| ControlError::Invalid("concurrency claim has no group".into()))?; + capacity + .held + .entry(id) + .or_default() + .insert(ConcurrencyResource::Limit(key.into())); + } + for child in runs + .iter() + .filter(|run| run.request_json.kind != RunKind::Group) + { + let Some(parent) = child + .request_json + .parent_run_id + .and_then(|id| parents.get(&id)) + else { + continue; + }; + if parent.state.cancel_requested_at.is_some() + || (parent.state.status == RunStatus::Queued + && !capacity + .waiting + .iter() + .any(|waiter| waiter.request_json.run_id == parent.request_json.run_id)) + { + continue; + } + let inherited = if parent.state.status == RunStatus::Running { + capacity.held[&parent.request_json.run_id].clone() + } else { + parent_resources(&parent.request_json) + }; + capacity + .needed + .entry(parent.request_json.run_id) + .or_default() + .extend(claim_resources(&child.request_json, Some(&inherited))); + } + for resource in capacity + .held + .values() + .flatten() + .chain(capacity.needed.values().flatten()) + { + if !capacity.available.contains_key(resource) { + match resource { + ConcurrencyResource::Overlap(..) => { + capacity.available.insert(resource.clone(), 1); + } + ConcurrencyResource::Limit(key) => { + return Err(ControlError::Invalid(format!( + "concurrency group {key} has no configured limit" + ))); + } + } + } + } + for resource in capacity.held.values().flatten() { + *capacity.available.entry(resource.clone()).or_insert(0) -= 1; + } + Ok((transaction, capacity)) +} + +/// Give earlier executable contenders first claim, without blocking children that drain parents. +pub(crate) async fn earlier_waiter_blocks( + mut transaction: ControlTransaction, + capacity: &GroupCapacity, + candidate: &QueueCandidate, +) -> Result<(ControlTransaction, bool), ControlError> { + if candidate.request_json.parent_run_id.is_some() { + return Ok((transaction, false)); + } + let footprint = parent_resources(&candidate.request_json); + for older in capacity.earlier_waiters(candidate) { + let inherited = older + .request_json + .parent_run_id + .and_then(|id| capacity.held.get(&id)); + let mut waiting = claim_resources(&older.request_json, inherited); + if candidate.request_json.kind == RunKind::Group { + waiting.extend( + capacity + .needed + .get(&older.request_json.run_id) + .into_iter() + .flatten() + .cloned(), + ); + } + if footprint.is_disjoint(&waiting) { + continue; + } + let ready; + (transaction, ready) = child_ready(transaction, &older.request_json).await?; + if ready { + return Ok((transaction, true)); + } + } + Ok((transaction, false)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs new file mode 100644 index 0000000000..23bc9fb8b2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/capacity.rs @@ -0,0 +1,121 @@ +//! Completion-order safety for persistent parent claims and atomic, temporary child claims. + +use std::cmp::Reverse; +use std::collections::BTreeSet; + +use crate::runs::models::{ConcurrencyResource, GroupCapacity, QueueCandidate, RunRequest}; +use crate::scheduler::models::Overlap; + +pub(crate) fn parent_resources(request: &RunRequest) -> BTreeSet { + let mut resources: BTreeSet<_> = request + .concurrency_groups + .iter() + .cloned() + .map(ConcurrencyResource::Limit) + .collect(); + if let Some(key) = &request.job_key { + resources.insert(ConcurrencyResource::Overlap( + request.location_id, + request.target.clone(), + key.clone(), + )); + } + resources +} + +pub(crate) fn claim_resources( + request: &RunRequest, + inherited: Option<&BTreeSet>, +) -> BTreeSet { + parent_resources(request) + .into_iter() + .filter(|resource| match resource { + ConcurrencyResource::Limit(_) => inherited.is_none_or(|held| !held.contains(resource)), + ConcurrencyResource::Overlap(..) => request.overlap != Overlap::Allow, + }) + .collect() +} + +impl GroupCapacity { + /// Find a completion order without reserving children's independent limits. + pub(crate) fn safe(&self, candidate: &RunRequest) -> bool { + let mut work = self.available.clone(); + let mut held = self.held.clone(); + let resources = parent_resources(candidate); + for resource in &resources { + *work.entry(resource.clone()).or_insert(1) -= 1; + } + held.insert(candidate.run_id, resources); + while !held.is_empty() { + let before = held.len(); + held.retain(|id, resources| { + if self.needed[id] + .iter() + .any(|resource| work.get(resource).copied().unwrap_or(1) <= 0) + { + return true; + } + for resource in resources.iter() { + *work.entry(resource.clone()).or_insert(0) += 1; + } + false + }); + if held.len() == before { + return false; + } + } + true + } + + pub(crate) fn blocks_claim( + &self, + candidate: &QueueCandidate, + waiting: &BTreeSet, + ) -> bool { + let inherited = candidate + .request_json + .parent_run_id + .and_then(|id| self.held.get(&id)); + let mut consumed = claim_resources(&candidate.request_json, inherited); + consumed.extend( + parent_resources(&candidate.request_json) + .into_iter() + .filter(|resource| matches!(resource, ConcurrencyResource::Overlap(..))), + ); + !consumed.is_disjoint(waiting) + } + + /// Only temporary blockers drain ahead of younger work; persistent blockers must let owners finish. + pub(crate) fn record_waiter( + &self, + candidate: &RunRequest, + mut waiting: BTreeSet, + ) -> BTreeSet { + let inherited = candidate.parent_run_id.and_then(|id| self.held.get(&id)); + let resources = claim_resources(candidate, inherited); + if resources.iter().all(|resource| { + self.available + .get(resource) + .is_none_or(|available| *available > 0) + }) { + waiting.extend(resources); + } + waiting + } + + pub(crate) fn earlier_waiters<'a>( + &'a self, + candidate: &'a QueueCandidate, + ) -> impl Iterator { + let rank = |value: &QueueCandidate| { + ( + Reverse(value.request_json.priority), + value.queued_at, + value.request_json.run_id, + ) + }; + self.waiting + .iter() + .filter(move |older| rank(older) < rank(candidate)) + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs new file mode 100644 index 0000000000..be4f644f81 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/context.rs @@ -0,0 +1,86 @@ +//! Read group membership and failure policy from the parent's immutable publication. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::declarations::models::OnFailure; +use crate::runs::models::{GroupContext, GroupRun, RunStatus}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn load_group( + transaction: ControlTransaction, + parent_id: Uuid, +) -> Result<(ControlTransaction, GroupContext), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1 AND r.kind='group'", + vec![Parameter::Id(parent_id)]).await?; + let parent: GroupRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("group run is missing".into()))?, + )?; + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at, r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.parent_run_id=$1 ORDER BY r.run_id", + vec![Parameter::Id(parent_id)]).await?; + let children = rows.into_iter().map(decode_row).collect::>()?; + let (transaction, definitions) = + frozen_definitions(transaction, parent.request_json.snapshot_id).await?; + let (transaction, failures) = query( + transaction, + "SELECT n.node_key FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE r.parent_run_id=$1 AND n.status='failed'", + vec![Parameter::Id(parent_id)], + ) + .await?; + let failures: BTreeSet = failures + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect(); + let (transaction, nodes) = read_snapshot_nodes( + transaction, + parent.request_json.snapshot_id, + Some(&failures), + ) + .await?; + let critical_failed = nodes + .values() + .any(|node| node.definition["config"]["critical"].as_bool() == Some(true)); + Ok(( + transaction, + GroupContext { + parent, + children, + definitions, + critical_failed, + }, + )) +} + +impl GroupContext { + pub(crate) fn stopped(&self) -> bool { + self.critical_failed + || self.children.iter().any(|child| { + child.request_json.job_key.as_ref().is_some_and(|name| { + self.definitions.jobs[name].on_failure == OnFailure::StopGroup + && (child + .state + .node_counts_json + .get("failed") + .copied() + .unwrap_or_default() + > 0 + || matches!( + child.state.status, + RunStatus::Failed | RunStatus::PresumedFailed + )) + }) + }) + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs new file mode 100644 index 0000000000..9e7cdb8fe2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs @@ -0,0 +1,206 @@ +//! A virtual parent advances only from durable child evidence under the scheduler's fence. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::admission::{earlier_waiter_blocks, group_capacity}; +use crate::runs::_helpers::groups::context::load_group; +use crate::runs::_helpers::lifecycle::claims::reserve_claims; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; +use crate::runs::models::{ + ChangingRun, GroupContext, NodeOutcome, QueueCandidate, RunChange, RunChangeRequest, + RunNodeRecord, RunOutcome, RunStatus, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Callers have already checked leadership; the surrounding transaction fences the entire advance. +pub(crate) async fn group_change( + transaction: ControlTransaction, + run_id: Uuid, + change: RunChange, +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", + vec![Parameter::Id(run_id)]).await?; + let state: ChangingRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("group run is missing".into()))?, + )?; + let event_id = Uuid::new_v5( + &run_id, + &serde_json::to_vec(&change).map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + publish_change( + transaction, + &RunChangeRequest { + event_id, + run_id, + claim_id: state.worker_instance, + change, + }, + &state, + ) + .await +} + +pub(crate) async fn advance_group( + transaction: ControlTransaction, + parent_id: Uuid, + limits: &BTreeMap, + eligible: Option<&BTreeSet<(Uuid, String)>>, +) -> Result { + let (mut transaction, group) = load_group(transaction, parent_id).await?; + let parent = &group.parent; + let children = &group.children; + if parent.state.status.is_terminal() { + return Ok(transaction); + } + let stopped = group.stopped(); + if parent.state.cancel_requested_at.is_some() || stopped { + for child in children { + if !child.state.status.is_terminal() && child.state.cancel_requested_at.is_none() { + transaction = group_change( + transaction, + child.request_json.run_id, + RunChange::CancelRequested { + actor: "system:group".into(), + reason: if stopped { + "group stopped after a child failure" + } else { + "parent group cancellation requested" + } + .into(), + }, + ) + .await?; + } + } + } else if parent.state.status == RunStatus::Queued { + let capacity; + (transaction, capacity) = group_capacity(transaction, limits, eligible).await?; + let blocked; + (transaction, blocked) = earlier_waiter_blocks( + transaction, + &capacity, + &QueueCandidate { + request_json: parent.request_json.clone(), + queued_at: parent.queued_at, + }, + ) + .await?; + if blocked || !capacity.safe(&parent.request_json) { + return Ok(transaction); + } + (transaction, _) = query(transaction, "SAVEPOINT group_claim", vec![]).await?; + let reserved; + (transaction, reserved) = reserve_claims(transaction, &parent.request_json, limits).await?; + if !reserved { + (transaction, _) = + query(transaction, "ROLLBACK TO SAVEPOINT group_claim", vec![]).await?; + } else { + transaction = group_change(transaction, parent_id, RunChange::Started).await?; + } + (transaction, _) = query(transaction, "RELEASE SAVEPOINT group_claim", vec![]).await?; + } + if !children.is_empty() + && children + .iter() + .all(|child| child.state.status.is_terminal()) + { + transaction = finish_group(transaction, &group, stopped).await?; + } + Ok(transaction) +} + +async fn finish_group( + transaction: ControlTransaction, + group: &GroupContext, + stopped: bool, +) -> Result { + let parent = &group.parent; + let parent_id = parent.request_json.run_id; + let (mut transaction, rows) = query(transaction, + "SELECT n.* FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id WHERE r.parent_run_id=$1 ORDER BY r.run_id, n.node_key", + vec![Parameter::Id(parent_id)]).await?; + let mut nodes: BTreeMap = BTreeMap::new(); + let rank = |status: &str| match status { + "failed" => 5, + "cancelled" => 4, + "skipped" => 3, + "warn" => 2, + "success" => 1, + _ => 0, + }; + for row in rows { + let node: RunNodeRecord = decode_row(row)?; + if nodes + .get(&node.node_key) + .is_none_or(|previous| rank(&node.status) > rank(&previous.status)) + { + nodes.insert(node.node_key.clone(), node); + } + } + let mut outcomes: BTreeSet = BTreeSet::new(); + for key in &parent.request_json.selection { + let node = nodes + .get(key) + .ok_or_else(|| ControlError::Invalid("group node has no child evidence".into()))?; + outcomes.insert(node.status.clone()); + let outcome: NodeOutcome = serde_json::from_value(serde_json::json!(node.status)) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + transaction = group_change( + transaction, + parent_id, + RunChange::NodeFinished { + node_key: key.clone(), + outcome, + rows_affected: node.rows_affected, + error_code: node.error_code.clone(), + skipped_because_of: node.skipped_because_of.clone(), + }, + ) + .await?; + } + let good = outcomes.contains("success") || outcomes.contains("warn"); + let bad = group + .children + .iter() + .any(|child| !matches!(child.state.status, RunStatus::Success | RunStatus::Warn)) + || outcomes + .iter() + .any(|status| !matches!(status.as_str(), "success" | "warn")); + let outcome = if parent.state.cancel_requested_at.is_some() { + RunOutcome::Cancelled + } else if stopped { + RunOutcome::Failed + } else if bad && good { + RunOutcome::Partial + } else if bad { + RunOutcome::Failed + } else if outcomes.contains("warn") + || group + .children + .iter() + .any(|child| child.state.status == RunStatus::Warn) + { + RunOutcome::Warn + } else { + RunOutcome::Success + }; + group_change( + transaction, + parent_id, + RunChange::Finished { + outcome, + reason: "all group children have durable terminal outcomes".into(), + exit_code: None, + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs new file mode 100644 index 0000000000..9f205ce914 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/mod.rs @@ -0,0 +1,5 @@ +pub(in crate::runs) mod admission; +pub(in crate::runs) mod capacity; +pub(in crate::runs) mod context; +pub(in crate::runs) mod lifecycle; +pub(in crate::runs) mod selection; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs new file mode 100644 index 0000000000..4f2c66310c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs @@ -0,0 +1,148 @@ +//! Failed nodes block only their lineage dependents unless an upstream job requests broader stops. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::models::OnFailure; +use crate::runs::_helpers::groups::context::load_group; +use crate::runs::models::{RunRequest, RunStatus}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn child_ready( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + let Some(parent_id) = request.parent_run_id else { + return Ok((transaction, true)); + }; + let (transaction, group) = load_group(transaction, parent_id).await?; + let name = request + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group child has no job".into()))?; + let parent_name = group + .parent + .request_json + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; + let dependencies = upstream_jobs(&group.definitions.groups[parent_name].jobs, name); + let ready = group.parent.state.status == RunStatus::Running + && group.parent.state.cancel_requested_at.is_none() + && !group.stopped() + && group.children.iter().all(|child| { + child + .request_json + .job_key + .as_ref() + .is_none_or(|name| !dependencies.contains(name.as_str())) + || child.state.status.is_terminal() + }); + Ok((transaction, ready)) +} + +pub(crate) async fn skipped_nodes( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + let Some(parent_id) = request.parent_run_id else { + return Ok((transaction, BTreeMap::new())); + }; + let (transaction, group) = load_group(transaction, parent_id).await?; + let name = request + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group child has no job".into()))?; + let parent_name = group + .parent + .request_json + .job_key + .as_ref() + .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; + let edges = &group.definitions.groups[parent_name].jobs; + let ancestors = upstream_jobs(edges, name); + let (transaction, rows) = query(transaction, + "SELECT n.node_key, n.status, n.skipped_because_of FROM sqb_run_nodes n JOIN sqb_runs r ON r.run_id=n.run_id \ + WHERE r.parent_run_id=$1 AND r.run_id<>$2 AND n.status IN ('failed','skipped','cancelled')", + vec![Parameter::Id(parent_id), Parameter::Id(request.run_id)]).await?; + let failures: BTreeMap = rows + .into_iter() + .filter_map(|row| { + let key = row.get("node_key")?.as_str()?.to_owned(); + let cause = row + .get("skipped_because_of") + .and_then(serde_json::Value::as_str) + .unwrap_or(&key) + .to_owned(); + Some((key, cause)) + }) + .collect(); + let whole_job = group.children.iter().find(|child| { + child.request_json.job_key.as_ref().is_some_and(|name| { + ancestors.contains(name.as_str()) + && group.definitions.jobs[name].on_failure == OnFailure::SkipDownstreamJobs + }) && child.state.status.is_terminal() + && !matches!(child.state.status, RunStatus::Success | RunStatus::Warn) + }); + if let Some(child) = whole_job { + let cause = child + .request_json + .selection + .iter() + .find_map(|key| failures.get(key)) + .cloned() + .unwrap_or_else(|| format!("run:{}", child.request_json.run_id)); + return Ok(( + transaction, + request + .selection + .iter() + .map(|key| (key.clone(), cause.clone())) + .collect(), + )); + } + if failures.is_empty() { + return Ok((transaction, BTreeMap::new())); + } + let (transaction, nodes) = read_snapshot_nodes(transaction, request.snapshot_id, None).await?; + let mut skipped: BTreeMap = BTreeMap::new(); + for selected in &request.selection { + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending = vec![selected.as_str()]; + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + if let Some(cause) = failures.get(key) { + skipped.insert(selected.clone(), cause.clone()); + break; + } + if let Some(node) = nodes.get(key) + && let Some(deps) = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + { + pending.extend(deps.iter().filter_map(serde_json::Value::as_str)); + } + } + } + Ok((transaction, skipped)) +} + +/// Completed jobs omitted from a retry do not sever explicit ordering between retained jobs. +fn upstream_jobs<'a>( + edges: &'a BTreeMap>, + name: &str, +) -> BTreeSet<&'a str> { + let mut ancestors: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = edges[name].iter().map(String::as_str).collect(); + while let Some(name) = pending.pop() { + if ancestors.insert(name) { + pending.extend(edges[name].iter().map(String::as_str)); + } + } + ancestors +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs new file mode 100644 index 0000000000..f6a099cf26 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs @@ -0,0 +1,175 @@ +//! Lifecycle authority checks precede fact publication and physical claim release. + +use crate::runs::models::{ + ChangingRun, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::types::Row; + +/// A launch-response observation is append-only even after completion, but requires recorded ownership. +pub(crate) async fn check_launch_claim( + transaction: ControlTransaction, + request: &RunChangeRequest, +) -> Result { + let claim_id = request.claim_id.ok_or_else(|| { + ControlError::Invalid("launch observation requires a recorded claim".into()) + })?; + let (transaction, rows) = query( + transaction, + "SELECT payload FROM recall_events WHERE event_id=$1 AND run_id=$2 \ + AND event_type='run_claimed' AND producer='scheduler' AND invocation_id=$3", + vec![ + Parameter::Text(claim_id.to_string()), + Parameter::Text(request.run_id.to_string()), + Parameter::Text(format!("scheduler/{}", request.run_id)), + ], + ) + .await?; + let recorded = rows.first().map(|row| &row["payload"]["claim"]); + if recorded.is_none_or(|claim| { + claim["claim_id"] != claim_id.to_string() + || claim["request"]["run_id"] != request.run_id.to_string() + }) { + return Err(ControlError::Invalid( + "launch observation does not belong to a recorded claim".into(), + )); + } + Ok(transaction) +} + +pub(crate) fn validate_retry( + previous: &Row, + request: &RunChangeRequest, + body: &serde_json::Value, +) -> Result<(), ControlError> { + if previous.get("payload") != Some(body) + || previous + .get("invocation_id") + .and_then(serde_json::Value::as_str) + != Some(format!("scheduler/{}", request.run_id).as_str()) + { + return Err(ControlError::Invalid( + "run change event ID already has different content".into(), + )); + } + Ok(()) +} + +pub(crate) fn validate_authority( + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result<(), ControlError> { + let allowed = match authority { + RunAuthority::Scheduler(_) => !matches!( + request.change, + RunChange::CancelRequested { .. } + | RunChange::Started + | RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + ), + RunAuthority::Worker { run_id, claim_id } => { + run_id == request.run_id + && request.claim_id == Some(claim_id) + && matches!( + request.change, + RunChange::Started + | RunChange::NodeStarted { .. } + | RunChange::NodeFinished { .. } + | RunChange::Recovered + | RunChange::Finished { + outcome: RunOutcome::Success + | RunOutcome::Warn + | RunOutcome::Partial + | RunOutcome::Failed + | RunOutcome::Cancelled, + .. + } + ) + } + RunAuthority::Operator(context) => { + matches!(&request.change, RunChange::CancelRequested { actor, reason } + if !context.actor.trim().is_empty() && context.actor == *actor + && !context.granting_role.trim().is_empty() && !reason.trim().is_empty()) + } + }; + if !allowed { + return Err(ControlError::Invalid( + "authority cannot publish this run change".into(), + )); + } + Ok(()) +} + +pub(crate) fn validate_change( + state: &ChangingRun, + request: &RunChangeRequest, +) -> Result<(), ControlError> { + use RunStatus::{Launched, Queued, Running, Unresponsive}; + if !matches!(state.status, Queued | Launched | Running | Unresponsive) { + return Err(ControlError::Invalid("terminal runs cannot change".into())); + } + if !matches!(request.change, RunChange::CancelRequested { .. }) + && request.claim_id != state.worker_instance + { + return Err(ControlError::Invalid( + "run change does not belong to the current claim".into(), + )); + } + let valid = match request.change { + RunChange::LaunchFailed { .. } => false, + RunChange::Launched { .. } => state.worker_instance.is_some(), + RunChange::Started => { + state.worker_instance.is_some() && matches!(state.status, Queued | Launched) + } + RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { + matches!(state.status, Running | Unresponsive) + } + RunChange::Unresponsive => state.status == Running, + RunChange::Recovered => state.status == Unresponsive, + RunChange::CancelRequested { .. } => state.cancel_requested_at.is_none(), + RunChange::Finished { outcome, .. } => { + state.worker_instance.is_some() + || matches!(outcome, RunOutcome::Failed | RunOutcome::Cancelled) + } + }; + if !valid { + return Err(ControlError::Invalid( + "run change is inconsistent with the current lifecycle".into(), + )); + } + if let RunChange::Finished { + outcome: RunOutcome::Success | RunOutcome::Warn | RunOutcome::Partial, + .. + } = request.change + { + let count = |status: &str| { + state + .node_counts_json + .get(status) + .copied() + .unwrap_or_default() + }; + let good = count("success") + count("warn"); + let bad = count("failed") + count("skipped") + count("cancelled"); + let expected = if bad > 0 && good > 0 { + RunOutcome::Partial + } else if bad > 0 { + RunOutcome::Failed + } else if count("warn") > 0 { + RunOutcome::Warn + } else { + RunOutcome::Success + }; + if count("pending") > 0 + || count("running") > 0 + || !matches!(request.change, RunChange::Finished { outcome, .. } if outcome == expected) + { + return Err(ControlError::Invalid( + "run outcome does not match its completed node facts".into(), + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs new file mode 100644 index 0000000000..0fcc9bebc1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/claims.rs @@ -0,0 +1,133 @@ +//! Reserve write claims and concurrency slots inside the candidate's rollback savepoint. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use serde_json::json; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::models::{RunClaim, RunRequest}; +use crate::scheduler::models::Overlap; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Occupied slots above a lowered limit still count against the run's concurrency group. +pub(crate) async fn reserve_claims( + mut transaction: ControlTransaction, + request: &RunRequest, + limits: &BTreeMap, +) -> Result<(ControlTransaction, bool), ControlError> { + if request.overlap != Overlap::Allow + && let Some(job_key) = &request.job_key + { + let rows; + (transaction, rows) = query( + transaction, + "SELECT r.run_id FROM sqb_runs r LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + AND (l.run_id IS NOT NULL OR (r.kind='group' AND r.status='running')) LIMIT 1", + vec![ + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + Parameter::Text(job_key.clone()), + ], + ) + .await?; + if !rows.is_empty() { + return Ok((transaction, false)); + } + } + let now = transaction.now; + for node in &request.write_set { + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_write_claims (target, node_key, run_id, claimed_at) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (target, node_key) DO NOTHING RETURNING node_key", + vec![Parameter::Text(request.target.clone()), Parameter::Text(node.clone()), Parameter::Id(request.run_id), Parameter::Time(now)]).await?; + if inserted.is_empty() { + return Ok((transaction, false)); + } + } + for group in &request.concurrency_groups { + if let Some(parent_id) = request.parent_run_id { + let inherited; + (transaction, inherited) = query( + transaction, + "SELECT slot FROM sqb_concurrency_slots WHERE group_key=$1 AND run_id=$2", + vec![Parameter::Text(group.clone()), Parameter::Id(parent_id)], + ) + .await?; + if !inherited.is_empty() { + continue; + } + } + let limit = limits + .get(group) + .ok_or_else(|| { + ControlError::Invalid(format!("concurrency group {group} has no configured limit")) + })? + .get(); + let slots; + (transaction, slots) = query( + transaction, + "SELECT slot FROM sqb_concurrency_slots WHERE group_key=$1 ORDER BY slot", + vec![Parameter::Text(group.clone())], + ) + .await?; + let occupied: BTreeSet = slots + .iter() + .filter_map(|row| row.get("slot").and_then(serde_json::Value::as_i64)) + .collect(); + if occupied.len() >= limit as usize { + return Ok((transaction, false)); + } + let slot = (1..=i64::from(limit)) + .find(|slot| !occupied.contains(slot)) + .ok_or_else(|| { + ControlError::Invalid("concurrency slot accounting is inconsistent".into()) + })?; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_concurrency_slots (group_key, slot, run_id, claimed_at) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (group_key, slot) DO NOTHING RETURNING slot", + vec![Parameter::Text(group.clone()), Parameter::Integer(slot), Parameter::Id(request.run_id), Parameter::Time(now)]).await?; + if inserted.is_empty() { + return Ok((transaction, false)); + } + } + Ok((transaction, true)) +} + +/// Publish ownership only after every physical claim has been reserved in this transaction. +pub(crate) async fn publish_claim( + transaction: ControlTransaction, + request: RunRequest, +) -> Result<(ControlTransaction, RunClaim), ControlError> { + let claim = RunClaim { + claim_id: Uuid::new_v5(&request.run_id, b"run_claimed"), + request, + }; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_run_liveness (run_id, worker_instance, claimed_at, last_heartbeat_at) VALUES ($1,$2,$3,$3)", + vec![Parameter::Id(claim.request.run_id), Parameter::Id(claim.claim_id), Parameter::Time(now)]).await?; + let (transaction, cursor) = append_run_event( + transaction, + (claim.request.run_id, claim.claim_id), + "run_claimed", + json!({"claim": claim}), + ) + .await?; + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![ + Parameter::Integer(cursor), + Parameter::Id(claim.request.run_id), + ], + ) + .await?; + Ok((transaction, claim)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs new file mode 100644 index 0000000000..a146a8e4a7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/events.rs @@ -0,0 +1,35 @@ +//! Lifecycle facts share Recall's invocation sequence and transaction. + +use serde_json::Value; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::models::{ControlEvent, ControlTransaction}; + +pub(crate) async fn append_run_event( + transaction: ControlTransaction, + (run_id, event_id): (Uuid, Uuid), + kind: &str, + payload: Value, +) -> Result<(ControlTransaction, i64), ControlError> { + let event = ControlEvent { + event_id, + invocation_id: format!("scheduler/{run_id}"), + event_type: kind, + producer: "scheduler", + occurred_at: transaction.now, + run_id: Some(run_id), + payload, + }; + let (transaction, outcome) = append_control_event(transaction, event).await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "run event conflicts with a durable fact".into(), + )); + } + let cursor = outcome + .storage_id + .ok_or_else(|| ControlError::Invalid("run event has no storage cursor".into()))?; + Ok((transaction, cursor)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs new file mode 100644 index 0000000000..0ef9c931db --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -0,0 +1,8 @@ +pub(in crate::runs) mod changes; +pub(in crate::runs) mod claims; +pub(in crate::runs) mod events; +pub(in crate::runs) mod nodes; +pub(in crate::runs) mod output; +pub(in crate::runs) mod projection; +pub(in crate::runs) mod publish_change; +pub(in crate::runs) mod recall; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs new file mode 100644 index 0000000000..6c609d9340 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/nodes.rs @@ -0,0 +1,84 @@ +//! Node lifecycle projections adjust bounded counters instead of rescanning nodes per event. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::runs::constants::{NODE_PENDING, NODE_RUNNING}; +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_node( + transaction: ControlTransaction, + request: &RunChangeRequest, + counts: &BTreeMap, +) -> Result { + let (node, status, rows, error, skipped) = match &request.change { + RunChange::NodeStarted { node_key } => (node_key, NODE_RUNNING, None, None, None), + RunChange::NodeFinished { + node_key, + outcome, + rows_affected, + error_code, + skipped_because_of, + } => ( + node_key, + outcome.as_str(), + *rows_affected, + error_code.clone(), + skipped_because_of.clone(), + ), + _ => { + return Err(ControlError::Invalid( + "node projection requires a node event".into(), + )); + } + }; + let now = transaction.now; + let (transaction, nodes) = query( + transaction, + "SELECT status FROM sqb_run_nodes WHERE run_id=$1 AND node_key=$2 AND attempt=1", + vec![Parameter::Id(request.run_id), Parameter::Text(node.clone())], + ) + .await?; + let previous = nodes + .first() + .and_then(|row| row.get("status")) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ControlError::Invalid("node is outside the frozen run selection".into()))?; + if previous != NODE_PENDING && (previous != NODE_RUNNING || status == NODE_RUNNING) { + return Err(ControlError::Invalid( + "node change is inconsistent with its current outcome".into(), + )); + } + let mut counts = counts.clone(); + let old = counts + .get_mut(previous) + .ok_or_else(|| ControlError::Invalid("node counter is missing".into()))?; + *old -= 1; + if *old < 0 { + return Err(ControlError::Invalid("node counter is negative".into())); + } + if *old == 0 { + counts.remove(previous); + } + *counts.entry(status.into()).or_default() += 1; + let (transaction, _) = query(transaction, + "UPDATE sqb_run_nodes SET status=$1, started_at=CASE WHEN $1='running' THEN $2 ELSE started_at END, \ + ended_at=$3, rows_affected=$4, error_code=$5, skipped_because_of=$6 WHERE run_id=$7 AND node_key=$8 AND attempt=1", + vec![Parameter::Text(status.into()), Parameter::Time(now), Parameter::OptionalTime((status != NODE_RUNNING).then_some(now)), + rows.map_or(Parameter::Null(sqlbuild_recall::schema::models::ColumnType::Integer), Parameter::Integer), + Parameter::OptionalText(error), Parameter::OptionalText(skipped), Parameter::Id(request.run_id), Parameter::Text(node.clone())]).await?; + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET node_counts_json=$1 WHERE run_id=$2", + vec![ + Parameter::Json(json!(counts)), + Parameter::Id(request.run_id), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs new file mode 100644 index 0000000000..80f466411a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/output.rs @@ -0,0 +1,80 @@ +//! Normalize Recall output under its physical claim's invocation envelope. + +use chrono::{DateTime, SecondsFormat}; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{OutputChunk, RecallRow, RowEnvelope}; +use uuid::Uuid; + +use crate::runs::constants::{OUTPUT_BATCH_BYTES, OUTPUT_BATCH_CHUNKS}; +use crate::runs::models::ClaimMetadata; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn claim_envelope( + transaction: ControlTransaction, + (run_id, claim_id): (Uuid, Uuid), +) -> Result<(ControlTransaction, RowEnvelope), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT l.claimed_at, s.sqb_version FROM sqb_run_liveness l \ + JOIN sqb_runs r ON r.run_id=l.run_id JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ + WHERE l.run_id=$1 AND l.worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ) + .await?; + let metadata: ClaimMetadata = decode_row(rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("publication does not belong to an active run claim".into()) + })?)?; + Ok(( + transaction, + RowEnvelope { + invocation_id: format!("worker/{claim_id}"), + invocation_started_at: metadata + .claimed_at + .to_rfc3339_opts(SecondsFormat::Micros, true), + producer_version: metadata.sqb_version, + }, + )) +} + +/// Keep MAX(sequence) + 1 representable for Recall completeness accounting. +pub(crate) fn output_rows( + envelope: &RowEnvelope, + chunks: &[OutputChunk], +) -> Result, ControlError> { + if chunks.len() > OUTPUT_BATCH_CHUNKS + || chunks.iter().map(|chunk| chunk.data.len()).sum::() > OUTPUT_BATCH_BYTES + { + return Err(ControlError::Invalid( + "output batch exceeds its limit".into(), + )); + } + let mut rows = Vec::with_capacity(chunks.len()); + for chunk in chunks { + let mut chunk = chunk.clone(); + let mut integers = vec![chunk.sequence, chunk.byte_offset, chunk.dropped_before]; + if let Some(omitted) = chunk.omitted { + integers.extend([omitted.bytes, omitted.sequences]); + if !chunk.data.is_empty() || omitted.sequences == 0 { + return Err(ControlError::Invalid( + "invalid output omission marker".into(), + )); + } + } + if chunk.sequence >= i64::MAX as u64 + || integers.iter().any(|value| i64::try_from(*value).is_err()) + { + return Err(ControlError::Invalid( + "output counter is out of range".into(), + )); + } + chunk.occurred_at = DateTime::parse_from_rfc3339(&chunk.occurred_at) + .map_err(|_| ControlError::Invalid("invalid output timestamp".into()))? + .to_utc() + .to_rfc3339_opts(SecondsFormat::Micros, true); + rows.push(output_row(envelope, &chunk)); + } + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs new file mode 100644 index 0000000000..b430f35721 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs @@ -0,0 +1,116 @@ +//! Lifecycle projections and claim cleanup are part of the fact's transaction, never a later task. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::runs::_helpers::lifecycle::nodes::project_node; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn project_change( + mut transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + let now = transaction.now; + match &request.change { + RunChange::LaunchFailed { .. } => {} + RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { + return project_node(transaction, request, &state.node_counts_json).await; + } + RunChange::Launched { container_id } => { + (transaction, _) = query(transaction, + "UPDATE sqb_runs SET status=CASE WHEN status='queued' THEN 'launched' ELSE status END, launched_at=COALESCE(launched_at,$1) WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + (transaction, _) = query( + transaction, + "UPDATE sqb_run_liveness SET container_id=$1 WHERE run_id=$2", + vec![ + Parameter::OptionalText(container_id.clone()), + Parameter::Id(request.run_id), + ], + ) + .await?; + } + RunChange::Started => { + (transaction, _) = query(transaction, + "UPDATE sqb_runs SET status='running', launched_at=COALESCE(launched_at,$1), started_at=$1 WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + } + RunChange::Unresponsive | RunChange::Recovered => { + let status = if request.change == RunChange::Unresponsive { + "unresponsive" + } else { + "running" + }; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status=$1 WHERE run_id=$2", + vec![ + Parameter::Text(status.into()), + Parameter::Id(request.run_id), + ], + ) + .await?; + } + RunChange::CancelRequested { .. } => { + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET cancel_requested_at=$1 WHERE run_id=$2", + vec![Parameter::Time(now), Parameter::Id(request.run_id)], + ) + .await?; + } + RunChange::Finished { + outcome, reason, .. + } => { + let cancelled = *outcome == RunOutcome::Cancelled; + (transaction, _) = query(transaction, + "UPDATE sqb_run_nodes SET status=CASE WHEN status='running' THEN $1 ELSE $2 END, ended_at=$3 \ + WHERE run_id=$4 AND status IN ('pending','running')", + vec![Parameter::Text(if cancelled { "cancelled" } else { "failed" }.into()), + Parameter::Text(if cancelled { "cancelled" } else { "skipped" }.into()), Parameter::Time(now), Parameter::Id(request.run_id)]).await?; + let counts; + (transaction, counts) = query(transaction, + "SELECT status, COUNT(*) AS count FROM sqb_run_nodes WHERE run_id=$1 GROUP BY status", + vec![Parameter::Id(request.run_id)]).await?; + let counts: BTreeMap = counts + .into_iter() + .filter_map(|row| { + Some(( + row.get("status")?.as_str()?.into(), + row.get("count")?.as_i64()?, + )) + }) + .collect(); + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status=$1, status_reason=$2, ended_at=$3, node_counts_json=$5 WHERE run_id=$4", + vec![ + Parameter::Text(outcome.as_str().into()), + Parameter::Text(reason.clone()), + Parameter::Time(now), + Parameter::Id(request.run_id), + Parameter::Json(json!(counts)), + ], + ) + .await?; + for table in [ + "sqb_write_claims", + "sqb_concurrency_slots", + "sqb_run_liveness", + ] { + (transaction, _) = query( + transaction, + &format!("DELETE FROM {table} WHERE run_id=$1"), + vec![Parameter::Id(request.run_id)], + ) + .await?; + } + } + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs new file mode 100644 index 0000000000..c5445432e7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -0,0 +1,95 @@ +//! Publish facts atomically; launch observations never project state or release physical ownership. + +use serde_json::json; + +use crate::platform::models::PlatformEvent; +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::_helpers::lifecycle::projection::project_change; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +pub(crate) async fn publish_change( + transaction: ControlTransaction, + request: &RunChangeRequest, + state: &ChangingRun, +) -> Result { + let (mut transaction, mut cursor) = append_run_event( + transaction, + (request.run_id, request.event_id), + request.change.event_type(), + json!({"change": request}), + ) + .await?; + if let (RunChange::LaunchFailed { code }, Some(claim_id)) = (&request.change, request.claim_id) + { + let platform = PlatformEvent::LaunchFailed { + run_id: request.run_id, + claim_id, + code: code.clone(), + }; + let occurred_at = transaction.now; + let outcome; + (transaction, outcome) = append_control_event( + transaction, + ControlEvent { + event_id: uuid::Uuid::new_v5(&request.event_id, b"platform-launch-failed"), + invocation_id: format!("platform/{}", request.run_id), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: Some(request.run_id), + payload: json!({"platform": platform}), + }, + ) + .await?; + if outcome.conflict.is_some() { + return Err(ControlError::Invalid( + "launch failure conflicts with a durable fact".into(), + )); + } + return Ok(transaction); + } + let mut transaction = project_change(transaction, request, state).await?; + let mut cancel_unstarted = matches!(request.change, RunChange::CancelRequested { .. }) + && state.worker_instance.is_none(); + if cancel_unstarted { + let rows; + (transaction, rows) = query( + transaction, + "SELECT kind FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + cancel_unstarted = rows.first().and_then(|row| row["kind"].as_str()) != Some("group"); + } + if cancel_unstarted { + let terminal = RunChangeRequest { + event_id: uuid::Uuid::new_v5(&request.event_id, b"cancelled-before-launch"), + run_id: request.run_id, + claim_id: None, + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "cancelled before launch".into(), + exit_code: None, + }, + }; + (transaction, cursor) = append_run_event( + transaction, + (terminal.run_id, terminal.event_id), + terminal.change.event_type(), + json!({"change": terminal}), + ) + .await?; + transaction = project_change(transaction, &terminal, state).await?; + } + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(request.run_id)], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs new file mode 100644 index 0000000000..ecd6740f79 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/recall.rs @@ -0,0 +1,103 @@ +//! Namespace Recall facts and validate artifact ownership before publication. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{RecallRow, RecallValue, RowEnvelope}; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::store::models::InsertOutcome; + +use crate::runs::models::RecallRecord; +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Recall statement projections join correlations globally, not only by invocation. +pub(crate) fn claim_record( + envelope: &RowEnvelope, + record: &RecallRecord, +) -> Result { + let mut values = record.values.clone(); + for name in [ + "event_id", + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", + ] { + if let Some(RecallValue::Text(id)) = values.get_mut(name) { + *id = serde_json::json!([envelope.invocation_id, id]).to_string(); + } + } + match record.table { + Table::Meta | Table::Conflicts => { + return Err(ControlError::Invalid( + "Recall store metadata is server-owned".into(), + )); + } + Table::Artifacts => { + let Some(RecallValue::Blob(content)) = values.get("content") else { + return Err(ControlError::Invalid( + "Recall artifact content must be bytes".into(), + )); + }; + if values.get("digest") + != Some(&RecallValue::Text(format!( + "sha256:{}", + sha256_hex(content) + ))) + || values.get("size") != Some(&RecallValue::Integer(content.len() as i64)) + { + return Err(ControlError::Invalid( + "Recall artifact digest or size differs from its bytes".into(), + )); + } + } + _ => {} + } + RecallRow::from_values(envelope, record.table, &values) + .map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) async fn append_records( + mut transaction: ControlTransaction, + envelope: &RowEnvelope, + rows: &[RecallRow], +) -> Result<(ControlTransaction, InsertOutcome), ControlError> { + let mut result = InsertOutcome::default(); + let mut delivered: BTreeMap = BTreeMap::new(); + for row in rows { + if row.table == Table::ArtifactRefs + && !rows.iter().any(|candidate| { + candidate.table == Table::Artifacts + && candidate.value("digest") == row.value("digest") + }) + { + let Some(RecallValue::Text(digest)) = row.value("digest") else { + return Err(ControlError::Invalid( + "Recall artifact digest must be text".into(), + )); + }; + let references; + (transaction, references) = query(transaction, + "SELECT 1 AS present FROM recall_artifact_refs WHERE invocation_id=$1 AND digest=$2 LIMIT 1", + vec![Parameter::Text(envelope.invocation_id.clone()), Parameter::Text(digest.clone())]).await?; + if references.is_empty() { + return Err(ControlError::Invalid( + "Recall artifact reference has no invocation-owned content".into(), + )); + } + } + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + if let Some(conflict) = outcome.conflict { + result.conflicts.push(conflict); + } else { + *delivered.entry(row.table).or_insert(0) += 1; + } + } + result.delivered = delivered.into_iter().collect(); + Ok((transaction, result)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs new file mode 100644 index 0000000000..5d47d7cf23 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod groups; +pub(crate) mod lifecycle; +pub(crate) mod requests; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs new file mode 100644 index 0000000000..a2c223fbb9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -0,0 +1,63 @@ +//! Freeze execution inputs from one published job or group in the caller's write transaction. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::runs::main::job_write_set::job_write_set; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn frozen_group_children( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, definitions) = frozen_definitions(transaction, request.snapshot_id).await?; + let group = request + .job_key + .as_ref() + .and_then(|name| definitions.groups.get(name)) + .ok_or_else(|| ControlError::Invalid("group is absent from the pinned snapshot".into()))?; + let selection_matches = if request.origin == RunOrigin::Rerun { + request.selection.is_subset(&group.job.selection) + } else { + group.job.selection == request.selection + }; + if !selection_matches || !request.write_set.is_empty() { + return Err(ControlError::Invalid( + "group scope differs from its pinned membership".into(), + )); + } + let (transaction, writers) = + job_write_set(transaction, request.snapshot_id, &request.selection).await?; + let mut children: Vec = Vec::new(); + for name in group.jobs.keys() { + let job = &definitions.jobs[name]; + let selection: BTreeSet<_> = job + .selection + .intersection(&request.selection) + .cloned() + .collect(); + if selection.is_empty() { + continue; + } + children.push(RunRequest { + run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), + kind: RunKind::Job, + job_key: Some(name.clone()), + parent_run_id: Some(request.run_id), + rerun_of_run_id: request + .rerun_of_run_id + .map(|id| Uuid::new_v5(&id, name.as_bytes())), + write_set: selection.intersection(&writers).cloned().collect(), + selection, + settings: job.settings.clone(), + concurrency_groups: job.concurrency_groups.clone(), + overlap: job.overlap, + ..request.clone() + }); + } + Ok((transaction, children)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs new file mode 100644 index 0000000000..010bce9e3a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs @@ -0,0 +1,3 @@ +pub(in crate::runs) mod job; +pub(in crate::runs) mod publication; +pub(in crate::runs) mod rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs new file mode 100644 index 0000000000..5f6a1eae99 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs @@ -0,0 +1,100 @@ +//! Validate and publish frozen requests inside the caller's audit transaction. + +use std::collections::BTreeSet; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::RunRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Check the original authored request before reading mutable publication pointers or outcomes. +pub(crate) async fn request_retry( + transaction: ControlTransaction, + context: &AuditContext, + (run_id, event_id, request): (Uuid, Uuid, &serde_json::Value), +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, previous) = query( + transaction, + "SELECT a.actor, a.after_json FROM sqb_runs r LEFT JOIN sqb_security_audit a ON a.event_id=$1 WHERE r.run_id=$2", + vec![Parameter::Id(event_id), Parameter::Id(run_id)], + ).await?; + let Some(previous) = previous.first() else { + return Ok((transaction, false)); + }; + if previous.get("actor") != Some(&json!(context.actor)) + || previous + .get("after_json") + .and_then(|after| after.get("request")) + != Some(request) + { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + Ok((transaction, true)) +} + +pub(crate) async fn publish_request( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, bool), ControlError> { + if request.requested_by.trim().is_empty() + || request.selection.is_empty() + || !request.write_set.is_subset(&request.selection) + { + return Err(ControlError::Invalid( + "run needs an actor, selection and a contained write set".into(), + )); + } + let (transaction, previous) = query( + transaction, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + if let Some(previous) = previous.first() { + if previous.get("request_json") != Some(&json!(request)) { + return Err(ControlError::Invalid( + "run ID already has different request content".into(), + )); + } + return Ok((transaction, false)); + } + if request.parent_run_id.is_some() { + return Err(ControlError::Invalid( + "group children are created atomically with their pinned parent".into(), + )); + } + let (transaction, nodes) = query( + transaction, + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND EXISTS \ + (SELECT 1 FROM sqb_snapshots WHERE snapshot_id=$1 AND location_id=$2 AND target=$3 \ + AND image_id IS NOT DISTINCT FROM $4)", + vec![ + Parameter::Id(request.snapshot_id), + Parameter::Id(request.location_id), + Parameter::Text(request.target.clone()), + Parameter::OptionalText(request.image_id.clone()), + ], + ) + .await?; + let keys: BTreeSet<&str> = nodes + .iter() + .filter_map(|row| row.get("node_key").and_then(serde_json::Value::as_str)) + .collect(); + if request + .selection + .iter() + .any(|key| !keys.contains(key.as_str())) + { + return Err(ControlError::Invalid( + "run selection or image does not match the pinned location and target snapshot".into(), + )); + } + Ok((queue_run(transaction, request).await?, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs new file mode 100644 index 0000000000..cd1e9a7ecc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs @@ -0,0 +1,68 @@ +//! Retry immutable execution inputs, narrowing only to durable failed and skipped outcomes. + +use std::collections::BTreeSet; + +use serde_json::from_value; + +use crate::credentials::models::AuditContext; +use crate::runs::models::{RerunRequest, RunOrigin, RunRequest, RunStatus}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_rerun( + transaction: ControlTransaction, + context: &AuditContext, + request: &RerunRequest, +) -> Result<(ControlTransaction, RunRequest), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT request_json, status FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.source_run_id)], + ) + .await?; + let source = rows + .first() + .ok_or_else(|| ControlError::Invalid("source run was not found".into()))?; + let status: RunStatus = from_value(source["status"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if !status.is_terminal() { + return Err(ControlError::Invalid( + "rerun requires a terminal source run".into(), + )); + } + let mut frozen: RunRequest = from_value(source["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.from_failure { + let rows; + (transaction, rows) = query( + transaction, + "SELECT node_key FROM sqb_run_nodes WHERE run_id=$1 AND status IN ('failed','skipped')", + vec![Parameter::Id(request.source_run_id)], + ) + .await?; + let failed: BTreeSet<_> = rows + .iter() + .filter_map(|row| row["node_key"].as_str()) + .collect(); + frozen.selection.retain(|key| failed.contains(key.as_str())); + frozen + .write_set + .retain(|key| frozen.selection.contains(key)); + if frozen.selection.is_empty() { + return Err(ControlError::Invalid( + "source run has no failed or skipped nodes".into(), + )); + } + } + frozen.run_id = request.run_id; + frozen.origin = RunOrigin::Rerun; + frozen.parent_run_id = None; + frozen.root_run_id = request.run_id; + frozen.trigger_id = None; + frozen.tick_id = None; + frozen.rerun_of_run_id = Some(request.source_run_id); + frozen.requested_by = context.actor.clone(); + frozen.priority = request.priority; + Ok((transaction, frozen)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs new file mode 100644 index 0000000000..661f08ec4f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -0,0 +1,12 @@ +//! Nonterminal node states; terminal states come from NodeOutcome. + +pub(crate) const NODE_PENDING: &str = "pending"; +pub(crate) const NODE_RUNNING: &str = "running"; + +pub(crate) const HEARTBEAT_TIMEOUT_SECONDS: i64 = 45; +pub(crate) const MONITOR_BATCH_SIZE: usize = 100; + +pub(crate) const OUTPUT_BATCH_CHUNKS: usize = 256; +pub(crate) const OUTPUT_BATCH_BYTES: usize = 1_048_576; +pub(crate) const RECALL_BATCH_ROWS: usize = 500; +pub(crate) const RECALL_INVOCATION_ID_BYTES: usize = 256; diff --git a/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs b/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs new file mode 100644 index 0000000000..33b0e9320e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/advance_groups.rs @@ -0,0 +1,51 @@ +//! Resume virtual parents independently of launcher capacity or process restarts. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; +use std::time::{Duration, Instant}; + +use crate::runs::_helpers::groups::lifecycle::advance_group; +use crate::runs::models::RunDispatch; +use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; +use crate::scheduler::main::{check_lease::check_lease, renew_lease::renew_lease}; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn advance_groups<'a>( + store: &ControlStore, + lease: &SchedulerLease, + (limits, dispatch): (&BTreeMap, impl Into>), +) -> Result<(), ControlError> { + let dispatch = dispatch.into(); + let scope = dispatch.scope.ok_or_else(|| { + ControlError::Invalid("group advancement requires a dispatch scope".into()) + })?; + let mut after = None; + let mut renewed = Instant::now(); + loop { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT run_id FROM sqb_runs WHERE kind='group' AND status IN ('queued','running') \ + AND location_id=$1 AND target=$2 AND ($3 IS NULL OR run_id>$3) ORDER BY run_id LIMIT 100", + vec![Parameter::Id(scope.0), Parameter::Text(scope.1.into()), Parameter::OptionalId(after)]).await?; + commit(transaction).await?; + if rows.is_empty() { + return Ok(()); + } + for row in rows { + if renewed.elapsed() >= Duration::from_secs(RENEW_INTERVAL_SECONDS) { + renew_lease(store, lease).await?; + renewed = Instant::now(); + } + let run_id: uuid::Uuid = serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let transaction = + check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let transaction = advance_group(transaction, run_id, limits, dispatch.eligible).await?; + commit(check_lease(transaction, lease).await?).await?; + after = Some(run_id); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/change_run.rs b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs new file mode 100644 index 0000000000..59cb4f2a40 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/change_run.rs @@ -0,0 +1,88 @@ +//! Retry-safe facts, lifecycle projection and terminal claim release form one transaction. + +use serde_json::json; + +use crate::credentials::main::append_audit::append_audit; +use crate::runs::_helpers::lifecycle::changes::{ + check_launch_claim, validate_authority, validate_change, validate_retry, +}; +use crate::runs::_helpers::lifecycle::publish_change::publish_change; +use crate::runs::models::{ChangingRun, RunAuthority, RunChange, RunChangeRequest}; +use crate::scheduler::main::check_lease::check_lease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Use authenticated authority and record unclaimed cancellation as terminal in the same transaction. +pub async fn change_run( + store: &ControlStore, + authority: RunAuthority<'_>, + request: &RunChangeRequest, +) -> Result { + validate_authority(authority, request)?; + let mut transaction = begin(store, TransactionMode::Write).await?; + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + let body = json!({"change": request}); + let (transaction, previous) = query( + transaction, + "SELECT invocation_id, payload FROM recall_events WHERE event_id=$1 LIMIT 1", + vec![Parameter::Text(request.event_id.to_string())], + ) + .await?; + if let Some(previous) = previous.first() { + validate_retry(previous, request, &body)?; + commit(transaction).await?; + return Ok(false); + } + let (mut transaction, rows) = query( + transaction, + "SELECT r.status, r.cancel_requested_at, r.node_counts_json, l.worker_instance FROM sqb_runs r \ + LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let state: ChangingRun = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("run does not exist".into()))?, + )?; + if matches!(request.change, RunChange::LaunchFailed { .. }) { + transaction = check_launch_claim(transaction, request).await?; + } else { + validate_change(&state, request)?; + } + let mut transaction = publish_change(transaction, request, &state).await?; + if let (RunAuthority::Operator(context), RunChange::CancelRequested { reason, .. }) = + (authority, &request.change) + { + let mut rows; + (transaction, rows) = query( + transaction, + "SELECT location_id, target, status, cancel_requested_at FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let after = rows.remove(0); + transaction = append_audit( + transaction, + context, + (request.event_id, "run.cancel_requested"), + ( + json!({"location_id": after["location_id"], "target": after["target"], "run_id": request.run_id}), + json!({"status": state.status, "cancel_requested_at": state.cancel_requested_at}), + json!({"status": after["status"], "cancel_requested_at": after["cancel_requested_at"], "reason": reason}), + ), + ) + .await?; + } + if let RunAuthority::Scheduler(lease) = authority { + transaction = check_lease(transaction, lease).await?; + } + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs new file mode 100644 index 0000000000..41b5686001 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs @@ -0,0 +1,95 @@ +//! Claim priority-ordered work without allowing a blocked page to starve later candidates. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use sqlbuild_recall::schema::models::ColumnType; + +use crate::runs::_helpers::groups::admission::{earlier_waiter_blocks, group_capacity}; +use crate::runs::_helpers::groups::selection::child_ready; +use crate::runs::_helpers::lifecycle::claims::{publish_claim, reserve_claims}; +use crate::runs::models::{ConcurrencyResource, QueueCandidate, RunClaim, RunDispatch}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlStore, Parameter, TransactionMode}; + +/// Limits come from one installation configuration, never from competing per-job declarations. +pub async fn claim_next_run<'a>( + store: &ControlStore, + lease: &SchedulerLease, + limits: &BTreeMap, + dispatch: impl Into>, +) -> Result, ControlError> { + let dispatch = dispatch.into(); + let (mut transaction, capacity) = group_capacity( + check_lease(begin(store, TransactionMode::Write).await?, lease).await?, + limits, + dispatch.eligible, + ) + .await?; + let mut waiting: BTreeSet = BTreeSet::new(); + let locking = if matches!(transaction.connection, Some(Connection::Postgres(_))) { + " FOR UPDATE OF r SKIP LOCKED" + } else { + "" + }; + let mut after: Option = None; + loop { + let rows; + (transaction, rows) = query(transaction, + &format!("SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.kind<>'group' AND r.cancel_requested_at IS NULL \ + AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) \ + AND ($4 IS NULL OR (r.location_id=$4 AND r.target=$5)) \ + AND ($1 IS NULL OR r.priority<$1 OR (r.priority=$1 AND (r.queued_at>$2 OR (r.queued_at=$2 AND r.run_id>$3)))) \ + ORDER BY r.priority DESC, r.queued_at, r.run_id LIMIT 100{locking}"), + vec![after.as_ref().map_or(Parameter::Null(ColumnType::Integer), |value| Parameter::Integer(value.request_json.priority)), + Parameter::OptionalTime(after.as_ref().map(|value| value.queued_at)), + Parameter::OptionalId(after.as_ref().map(|value| value.request_json.run_id)), + Parameter::OptionalId(dispatch.scope.map(|(location, _)| location)), + Parameter::OptionalText(dispatch.scope.map(|(_, target)| target.to_owned()))]).await?; + if rows.is_empty() { + commit(check_lease(transaction, lease).await?).await?; + return Ok(None); + } + for row in rows { + let candidate: QueueCandidate = decode_row(row)?; + let ready; + (transaction, ready) = child_ready(transaction, &candidate.request_json).await?; + if !ready { + after = Some(candidate); + continue; + } + let blocked; + (transaction, blocked) = + earlier_waiter_blocks(transaction, &capacity, &candidate).await?; + if blocked || capacity.blocks_claim(&candidate, &waiting) { + after = Some(candidate); + continue; + } + (transaction, _) = query(transaction, "SAVEPOINT candidate_claim", vec![]).await?; + let reserved; + (transaction, reserved) = + reserve_claims(transaction, &candidate.request_json, limits).await?; + if !reserved { + (transaction, _) = + query(transaction, "ROLLBACK TO SAVEPOINT candidate_claim", vec![]).await?; + (transaction, _) = + query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + waiting = capacity.record_waiter(&candidate.request_json, waiting); + after = Some(candidate); + continue; + } + let claim; + (transaction, claim) = publish_claim(transaction, candidate.request_json).await?; + (transaction, _) = + query(transaction, "RELEASE SAVEPOINT candidate_claim", vec![]).await?; + commit(check_lease(transaction, lease).await?).await?; + return Ok(Some(claim)); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs new file mode 100644 index 0000000000..e91697d05d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -0,0 +1,70 @@ +//! Freeze published job inputs for manual requests and sensor evaluations alike. + +use crate::credentials::models::AuditContext; +use crate::declarations::models::JobDefinition; +use crate::runs::main::job_write_set::job_write_set; +use crate::runs::models::{JobRunRequest, RunKind, RunOrigin, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn frozen_job_request( + transaction: ControlTransaction, + context: &AuditContext, + request: &JobRunRequest, +) -> Result<(ControlTransaction, RunRequest), ControlError> { + let (transaction, jobs) = query( + transaction, + "SELECT j.snapshot_id, j.definition_json, j.kind, s.image_id FROM sqb_jobs j \ + JOIN sqb_current_snapshots c ON c.location_id=j.location_id AND c.target=j.target AND c.snapshot_id=j.snapshot_id \ + JOIN sqb_snapshots s ON s.snapshot_id=j.snapshot_id WHERE j.location_id=$1 AND j.target=$2 AND j.job_key=$3", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Text(request.job_key.clone())], + ).await?; + let row = jobs.first().ok_or_else(|| { + ControlError::Invalid("published job not found in this location and target".into()) + })?; + let snapshot_id = serde_json::from_value(row["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let definition: JobDefinition = serde_json::from_value(row["definition_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let image_id = serde_json::from_value(row["image_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let kind: RunKind = serde_json::from_value(row["kind"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let (transaction, write_set) = + job_write_set(transaction, snapshot_id, &definition.selection).await?; + Ok(( + transaction, + RunRequest { + run_id: request.run_id, + location_id: request.location_id, + target: request.target.clone(), + snapshot_id, + image_id, + kind, + origin: if request.window.is_some() { + RunOrigin::Backfill + } else { + RunOrigin::Manual + }, + job_key: Some(request.job_key.clone()), + parent_run_id: None, + root_run_id: request.run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: context.actor.clone(), + priority: request.priority, + selection: definition.selection, + write_set: if kind == RunKind::Group { + Default::default() + } else { + write_set + }, + settings: definition.settings, + window: request.window.clone(), + concurrency_groups: definition.concurrency_groups, + overlap: definition.overlap, + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs new file mode 100644 index 0000000000..dc0b6db2be --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/heartbeat_run.rs @@ -0,0 +1,71 @@ +//! Heartbeats update only liveness; recovery is a durable transition in that same transaction. + +use serde_json::json; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The worker identity is taken from its authenticated run token. Return whether cancellation is pending. +pub async fn heartbeat_run( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "UPDATE sqb_run_liveness SET last_heartbeat_at=$1 WHERE run_id=$2 AND worker_instance=$3 RETURNING run_id", + vec![Parameter::Time(now), Parameter::Id(run_id), Parameter::Id(claim_id)]).await?; + if rows.is_empty() { + return Err(ControlError::Invalid( + "heartbeat does not belong to an active run claim".into(), + )); + } + let (mut transaction, rows) = query( + transaction, + "SELECT status, last_event_storage_id, cancel_requested_at FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(run_id)], + ) + .await?; + let row = rows + .first() + .ok_or_else(|| ControlError::Invalid("claimed run does not exist".into()))?; + let cancelled = row + .get("cancel_requested_at") + .is_some_and(|value| !value.is_null()); + if row.get("status").and_then(serde_json::Value::as_str) == Some("unresponsive") { + let previous = row + .get("last_event_storage_id") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| { + ControlError::Invalid("run recovery is missing its preceding event".into()) + })?; + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim_id, format!("recovery/{previous}").as_bytes()), + run_id, + claim_id: Some(claim_id), + change: RunChange::Recovered, + }; + let cursor; + (transaction, cursor) = append_run_event( + transaction, + (run_id, request.event_id), + request.change.event_type(), + json!({"change": request}), + ) + .await?; + (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status='running', last_event_storage_id=$1 WHERE run_id=$2", + vec![Parameter::Integer(cursor), Parameter::Id(run_id)], + ) + .await?; + } + commit(transaction).await?; + Ok(cancelled) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs new file mode 100644 index 0000000000..91840568ee --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/job_write_set.rs @@ -0,0 +1,30 @@ +//! Resolve physical writers from the pinned snapshot and its published job membership. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn job_write_set( + transaction: ControlTransaction, + snapshot_id: Uuid, + selection: &BTreeSet, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND node_kind IN ('model','seed','udf','table_fn')", + vec![Parameter::Id(snapshot_id)]).await?; + Ok(( + transaction, + rows.into_iter() + .filter_map(|row| { + row.get("node_key") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + }) + .filter(|key| selection.contains(key)) + .collect(), + )) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs b/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs new file mode 100644 index 0000000000..216113dab9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_run_claims.rs @@ -0,0 +1,39 @@ +//! Recover ownership from active claims, not stale lifecycle events or elapsed heartbeat guesses. + +use uuid::Uuid; + +use crate::runs::constants::MONITOR_BATCH_SIZE; +use crate::runs::models::RunClaim; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Keyset pages include claimed-but-unstarted work and never include a neighboring scope. +pub async fn list_run_claims( + store: &ControlStore, + (location_id, target): (Uuid, &str), + after: Option, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + &format!( + "SELECT l.worker_instance AS claim_id, r.request_json AS request \ + FROM sqb_run_liveness l JOIN sqb_runs r ON r.run_id=l.run_id \ + WHERE r.location_id=$1 AND r.target=$2 AND ($3 IS NULL OR r.run_id>$3) \ + ORDER BY r.run_id LIMIT {MONITOR_BATCH_SIZE}" + ), + vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::OptionalId(after), + ], + ) + .await?; + let claims = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(claims) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs b/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs new file mode 100644 index 0000000000..f0312d6f5c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs @@ -0,0 +1,29 @@ +//! Node outcomes are scoped to one frozen run, independently of current code snapshots. + +use uuid::Uuid; + +use crate::runs::models::RunNodeRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The API authorizes visibility of the run before reading its nodes. +pub async fn list_run_nodes( + store: &ControlStore, + run_id: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT node_key, attempt, status, started_at, ended_at, rows_affected, error_code, skipped_because_of \ + FROM sqb_run_nodes WHERE run_id=$1 ORDER BY node_key, attempt", + vec![Parameter::Id(run_id)]).await?; + let nodes = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(nodes) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs new file mode 100644 index 0000000000..516302af18 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs @@ -0,0 +1,30 @@ +//! Installation API reads never mix locations or targets. + +use uuid::Uuid; + +use crate::runs::models::RunRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Return recent projected runs; per-run events remain in Recall. +pub async fn list_runs( + store: &ControlStore, + (location_id, target): (Uuid, &str), + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT run_id, location_id, target, snapshot_id, job_key, status, queued_at, launched_at, started_at, ended_at, selection_json, node_counts_json, last_event_storage_id \ + FROM sqb_runs WHERE location_id=$1 AND target=$2 ORDER BY queued_at DESC, run_id DESC LIMIT $3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let runs = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(runs) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/mod.rs b/crates/sqlbuild-orchestration/src/runs/main/mod.rs new file mode 100644 index 0000000000..6027f5ea35 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/mod.rs @@ -0,0 +1,17 @@ +pub mod advance_groups; +pub mod change_run; +pub mod claim_next_run; +pub(crate) mod frozen_job_request; +pub mod heartbeat_run; +pub(crate) mod job_write_set; +pub mod list_run_claims; +pub mod list_run_nodes; +pub mod list_runs; +pub mod monitor_runs; +pub mod observe_run; +pub mod publish_run_output; +pub mod publish_run_recall; +pub(crate) mod queue_run; +pub mod request_job_run; +pub mod request_rerun; +pub mod run_execution; diff --git a/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs new file mode 100644 index 0000000000..f6ea6f8420 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/monitor_runs.rs @@ -0,0 +1,55 @@ +//! Observe heartbeat loss under the scheduler fence without releasing a live worker's claims. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::publish_change::publish_change; +use crate::runs::constants::{HEARTBEAT_TIMEOUT_SECONDS, MONITOR_BATCH_SIZE}; +use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest}; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Serialized writers make heartbeat inspection and the resulting fact one atomic observation. +pub async fn monitor_runs( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let mut transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let cutoff = transaction.now - TimeDelta::seconds(HEARTBEAT_TIMEOUT_SECONDS); + let rows; + (transaction, rows) = query( + transaction, + &format!( + "SELECT r.run_id, r.status, r.cancel_requested_at, r.node_counts_json, \ + r.last_event_storage_id, l.worker_instance FROM sqb_runs r \ + JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + WHERE r.status='running' AND l.last_heartbeat_at<$1 \ + ORDER BY l.last_heartbeat_at, r.run_id LIMIT {MONITOR_BATCH_SIZE}" + ), + vec![Parameter::Time(cutoff)], + ) + .await?; + let observed = rows.len(); + for row in rows { + let run_id: Uuid = serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let previous: i64 = serde_json::from_value(row["last_event_storage_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let state: ChangingRun = decode_row(row)?; + let request = RunChangeRequest { + event_id: Uuid::new_v5(&run_id, format!("unresponsive/{previous}").as_bytes()), + run_id, + claim_id: state.worker_instance, + change: RunChange::Unresponsive, + }; + transaction = publish_change(transaction, &request, &state).await?; + } + commit(check_lease(transaction, lease).await?).await?; + Ok(observed) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs b/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs new file mode 100644 index 0000000000..f18e11f7a6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/observe_run.rs @@ -0,0 +1,24 @@ +//! Observe one run and its current claim using the same database-clock snapshot. + +use uuid::Uuid; + +use crate::runs::models::RunObservation; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, decode_row::decode_row, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn observe_run( + store: &ControlStore, + run_id: Uuid, +) -> Result { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, "SELECT r.status, r.cancel_requested_at, l.claimed_at, l.last_heartbeat_at, $2 AS observed_at FROM sqb_runs r LEFT JOIN sqb_run_liveness l ON l.run_id=r.run_id WHERE r.run_id=$1", vec![Parameter::Id(run_id), Parameter::Time(now)]).await?; + let state = decode_row( + rows.into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("launched run is missing".into()))?, + )?; + commit(transaction).await?; + Ok(state) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs new file mode 100644 index 0000000000..17e4e0c7a3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_output.rs @@ -0,0 +1,36 @@ +//! Exact-byte Recall output is namespaced by the authenticated physical claim, never a body ID. + +use sqlbuild_recall::records::models::OutputChunk; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::output::{claim_envelope, output_rows}; +use crate::runs::models::OutputPublication; +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +/// The active-claim check and the complete batch commit share a serialized write transaction. +pub async fn publish_run_output( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), + chunks: &[OutputChunk], +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (mut transaction, envelope) = claim_envelope(transaction, (run_id, claim_id)).await?; + let mut published = OutputPublication::default(); + for row in output_rows(&envelope, chunks)? { + let outcome; + (transaction, outcome) = append_recall(transaction, &row).await?; + if outcome.inserted { + published.inserted += 1; + } else if outcome.conflict.is_some() { + published.conflicts += 1; + } else { + published.duplicates += 1; + } + } + commit(transaction).await?; + Ok(published) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs new file mode 100644 index 0000000000..30c14bb9d8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/publish_run_recall.rs @@ -0,0 +1,39 @@ +//! A worker's complete Recall batch and active-claim check share one transaction. + +use sqlbuild_recall::store::models::InsertOutcome; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::output::claim_envelope; +use crate::runs::_helpers::lifecycle::recall::{append_records, claim_record}; +use crate::runs::constants::{RECALL_BATCH_ROWS, RECALL_INVOCATION_ID_BYTES}; +use crate::runs::models::RecallBatch; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn publish_run_recall( + store: &ControlStore, + identity: (Uuid, Uuid), + batch: &RecallBatch, +) -> Result { + if batch.invocation_id.trim().is_empty() + || batch.invocation_id.len() > RECALL_INVOCATION_ID_BYTES + || batch.records.len() > RECALL_BATCH_ROWS + { + return Err(ControlError::Invalid( + "invalid Recall invocation or batch size".into(), + )); + } + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, mut envelope) = claim_envelope(transaction, identity).await?; + envelope.invocation_id = format!("{}/{}", envelope.invocation_id, batch.invocation_id); + let rows = batch + .records + .iter() + .map(|record| claim_record(&envelope, record)) + .collect::, _>>()?; + let (transaction, result) = append_records(transaction, &envelope, &rows).await?; + commit(transaction).await?; + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs new file mode 100644 index 0000000000..fb4d86c590 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/queue_run.rs @@ -0,0 +1,73 @@ +//! Publish request facts, queued facts and initial node projections in the caller's transaction. + +use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::runs::_helpers::lifecycle::events::append_run_event; +use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::models::{RunKind, RunRequest}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn queue_run( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result { + let mut transaction = queue_one(transaction, request).await?; + if request.kind == RunKind::Group { + let children; + (transaction, children) = frozen_group_children(transaction, request).await?; + for child in children { + transaction = queue_one(transaction, &child).await?; + } + } + Ok(transaction) +} + +async fn queue_one( + mut transaction: ControlTransaction, + request: &RunRequest, +) -> Result { + let now = transaction.now; + let mut cursor = None; + for kind in ["run_requested", "run_queued"] { + let storage_id; + (transaction, storage_id) = append_run_event( + transaction, + ( + request.run_id, + Uuid::new_v5(&request.run_id, kind.as_bytes()), + ), + kind, + json!({"request": request}), + ) + .await?; + cursor = Some(storage_id); + } + let cursor = + cursor.ok_or_else(|| ControlError::Invalid("queued event has no storage cursor".into()))?; + let selection = serde_json::to_value(&request.selection) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let digest = sha256_hex( + &serde_json::to_vec(&request.write_set) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + (transaction, _) = query(transaction, + "INSERT INTO sqb_runs (run_id, location_id, target, snapshot_id, image_id, kind, origin, job_key, parent_run_id, root_run_id, trigger_id, tick_id, rerun_of_run_id, \ + requested_by, priority, selection_json, write_set_digest, status, queued_at, node_counts_json, last_event_storage_id, request_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'queued',$18,$19,$20,$21)", + vec![Parameter::Id(request.run_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), Parameter::Id(request.snapshot_id), + Parameter::OptionalText(request.image_id.clone()), Parameter::Text(request.kind.as_str().into()), Parameter::Text(request.origin.as_str().into()), + Parameter::OptionalText(request.job_key.clone()), Parameter::OptionalId(request.parent_run_id), Parameter::Id(request.root_run_id), + Parameter::OptionalId(request.trigger_id), Parameter::OptionalId(request.tick_id), Parameter::OptionalId(request.rerun_of_run_id), + Parameter::Text(request.requested_by.clone()), Parameter::Integer(request.priority), Parameter::Json(selection), Parameter::Text(digest), + Parameter::Time(now), Parameter::Json(json!({"pending": request.selection.len()})), Parameter::Integer(cursor), Parameter::Json(json!(request))]).await?; + for node in &request.selection { + (transaction, _) = query(transaction, + "INSERT INTO sqb_run_nodes (run_id, node_key, attempt, status, writes) VALUES ($1,$2,1,'pending',$3)", + vec![Parameter::Id(request.run_id), Parameter::Text(node.clone()), Parameter::Boolean(request.write_set.contains(node))]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs new file mode 100644 index 0000000000..8593e77c41 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -0,0 +1,66 @@ +//! Direct-store owners queue published jobs without supplying or widening their write scope. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; +use crate::runs::main::frozen_job_request::frozen_job_request; +use crate::runs::models::{JobRunRequest, RunWindow}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +/// The context is a trusted control-store owner, not a role supplied by an HTTP caller. +pub async fn request_job_run( + store: &ControlStore, + context: &AuditContext, + request: &JobRunRequest, +) -> Result { + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.target.trim().is_empty() + || request.job_key.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "job run needs an actor, granting role, target, job and reason".into(), + )); + } + if request.window.as_ref().is_some_and(|window| match window { + RunWindow::Timestamp { start, end } => start > end, + RunWindow::Integer { start, end } => start > end, + }) { + return Err(ControlError::Invalid( + "backfill window start must not follow end".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, b"manual-job-request"); + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, retry) = request_retry( + transaction, + context, + (request.run_id, event_id, &json!(request)), + ) + .await?; + if retry { + commit(transaction).await?; + return Ok(false); + } + let (transaction, frozen) = frozen_job_request(transaction, context, request).await?; + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = append_audit( + transaction, + context, + (event_id, "job.run_requested"), + ( + json!({"location_id": request.location_id, "target": request.target, "job_key": request.job_key, "run_id": request.run_id}), + serde_json::Value::Null, + json!({"request": request, "frozen": frozen}), + ), + ).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs new file mode 100644 index 0000000000..f44d096434 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/request_rerun.rs @@ -0,0 +1,54 @@ +//! Publish a new pinned execution and its operator audit atomically, including retry identity. + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::runs::_helpers::requests::publication::{publish_request, request_retry}; +use crate::runs::_helpers::requests::rerun::frozen_rerun; +use crate::runs::models::RerunRequest; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +/// This entry requires direct control-store ownership, just like a privileged manual job request. +pub async fn request_rerun( + store: &ControlStore, + context: &AuditContext, + request: &RerunRequest, +) -> Result { + if context.actor.trim().is_empty() + || context.granting_role.trim().is_empty() + || request.reason.trim().is_empty() + { + return Err(ControlError::Invalid( + "rerun needs an actor, granting role and reason".into(), + )); + } + let event_id = Uuid::new_v5(&request.run_id, b"rerun-request"); + let (transaction, retry) = request_retry( + begin(store, TransactionMode::Write).await?, + context, + (request.run_id, event_id, &json!(request)), + ) + .await?; + if retry { + commit(transaction).await?; + return Ok(false); + } + let (transaction, frozen) = frozen_rerun(transaction, context, request).await?; + let (transaction, _) = publish_request(transaction, &frozen).await?; + let transaction = append_audit( + transaction, + context, + (event_id, "run.rerun_requested"), + ( + json!({"location_id": frozen.location_id, "target": frozen.target, "job_key": frozen.job_key, "run_id": request.run_id}), + json!({"source_run_id": request.source_run_id}), + json!({"request": request, "frozen": frozen}), + ), + ).await?; + commit(transaction).await?; + Ok(true) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs b/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs new file mode 100644 index 0000000000..f8e49fc1f2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/main/run_execution.rs @@ -0,0 +1,61 @@ +//! Read the immutable execution contract only for the run's currently active claim. + +use std::collections::BTreeMap; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::selection::skipped_nodes; +use crate::runs::models::{ExecutionMetadata, RunClaim, RunExecution}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// A redeploy cannot change a claimed run's code or environment expectations. +pub async fn run_execution( + store: &ControlStore, + (run_id, claim_id): (Uuid, Uuid), +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT r.request_json, s.environment_hash, s.sqb_version FROM sqb_runs r \ + JOIN sqb_run_liveness l ON l.run_id=r.run_id \ + JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ + WHERE r.run_id=$1 AND l.worker_instance=$2", + vec![Parameter::Id(run_id), Parameter::Id(claim_id)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let metadata: ExecutionMetadata = decode_row(row)?; + let (transaction, rows) = query( + transaction, + "SELECT node_key, node_hash FROM sqb_snapshot_nodes WHERE snapshot_id=$1", + vec![Parameter::Id(metadata.request_json.snapshot_id)], + ) + .await?; + let node_hashes: BTreeMap = rows + .into_iter() + .map(|row| { + let fields: BTreeMap = decode_row(row)?; + Ok((fields["node_key"].clone(), fields["node_hash"].clone())) + }) + .collect::>()?; + let (transaction, skipped_nodes) = skipped_nodes(transaction, &metadata.request_json).await?; + commit(transaction).await?; + Ok(Some(RunExecution { + claim: RunClaim { + claim_id, + request: metadata.request_json, + }, + environment_hash: metadata.environment_hash, + sqb_version: metadata.sqb_version, + node_hashes, + skipped_nodes, + })) +} diff --git a/crates/sqlbuild-orchestration/src/runs/mod.rs b/crates/sqlbuild-orchestration/src/runs/mod.rs new file mode 100644 index 0000000000..529a59766a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/runs/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs new file mode 100644 index 0000000000..16625f5cc1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -0,0 +1,421 @@ +//! Frozen requests and rebuildable run projections; lifecycle authority remains in Recall. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunKind { + Job, + Group, + Adhoc, +} + +impl RunKind { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Job => "job", + Self::Group => "group", + Self::Adhoc => "adhoc", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunOrigin { + Manual, + Schedule, + Sensor, + Rerun, + Backfill, +} + +impl RunOrigin { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Manual => "manual", + Self::Schedule => "schedule", + Self::Sensor => "sensor", + Self::Rerun => "rerun", + Self::Backfill => "backfill", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunStatus { + Queued, + Launched, + Running, + Unresponsive, + Success, + Warn, + Partial, + Failed, + Cancelled, + PresumedFailed, +} + +impl RunStatus { + #[must_use] + pub const fn is_terminal(self) -> bool { + !matches!( + self, + Self::Queued | Self::Launched | Self::Running | Self::Unresponsive + ) + } +} + +/// Inclusive typed CLI cursor bounds; the native planner derives exclusive execution ends. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum RunWindow { + Timestamp { + start: DateTime, + end: DateTime, + }, + Integer { + start: i64, + end: i64, + }, +} + +/// The authorized selection and pinned execution inputs are immutable once requested. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunRequest { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub image_id: Option, + pub kind: RunKind, + pub origin: RunOrigin, + pub job_key: Option, + pub parent_run_id: Option, + pub root_run_id: Uuid, + pub trigger_id: Option, + pub tick_id: Option, + pub rerun_of_run_id: Option, + pub requested_by: String, + pub priority: i64, + pub selection: BTreeSet, + pub write_set: BTreeSet, + pub settings: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, + pub concurrency_groups: BTreeSet, + pub overlap: crate::scheduler::models::Overlap, +} + +/// A trusted operator names a published job; execution inputs are never supplied by the caller. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JobRunRequest { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub job_key: String, + pub reason: String, + pub priority: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, +} + +/// A retry derives its execution scope from a terminal run, never from caller selectors. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RerunRequest { + pub run_id: Uuid, + pub source_run_id: Uuid, + pub from_failure: bool, + pub reason: String, + pub priority: i64, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunRecord { + pub run_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub job_key: Option, + pub status: RunStatus, + pub queued_at: DateTime, + pub launched_at: Option>, + pub started_at: Option>, + pub ended_at: Option>, + pub selection_json: BTreeSet, + pub node_counts_json: BTreeMap, + pub last_event_storage_id: i64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum NodeOutcome { + Success, + Warn, + Failed, + Skipped, + Cancelled, +} + +impl NodeOutcome { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Warn => "warn", + Self::Failed => "failed", + Self::Skipped => "skipped", + Self::Cancelled => "cancelled", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunNodeRecord { + pub node_key: String, + pub attempt: i64, + pub status: String, + pub started_at: Option>, + pub ended_at: Option>, + pub rows_affected: Option, + pub error_code: Option, + pub skipped_because_of: Option, +} + +/// The database owns the claim before a launcher sees the pinned request. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunClaim { + pub claim_id: Uuid, + pub request: RunRequest, +} + +/// Workers compile locally and compare against this immutable snapshot, not the current deploy. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunExecution { + pub claim: RunClaim, + pub environment_hash: String, + pub sqb_version: String, + pub node_hashes: BTreeMap, + /// Derived from completed sibling facts and pinned lineage, never a mutable request rewrite. + #[serde(default)] + pub skipped_nodes: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ExecutionMetadata { + pub(crate) request_json: RunRequest, + pub(crate) environment_hash: String, + pub(crate) sqb_version: String, +} + +/// Terminal outcomes cannot be confused with a request to change an active status. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunOutcome { + Success, + Warn, + Partial, + Failed, + Cancelled, + PresumedFailed, +} + +impl RunOutcome { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Warn => "warn", + Self::Partial => "partial", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + Self::PresumedFailed => "presumed_failed", + } + } +} + +/// This is a server-owned instruction, not an arbitrary status supplied by a worker. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum RunChange { + Launched { + container_id: Option, + }, + /// A failed launch response is not physical evidence that the worker has stopped. + LaunchFailed { + code: String, + }, + Started, + NodeStarted { + node_key: String, + }, + NodeFinished { + node_key: String, + outcome: NodeOutcome, + rows_affected: Option, + error_code: Option, + skipped_because_of: Option, + }, + Unresponsive, + Recovered, + CancelRequested { + actor: String, + #[serde(default)] + reason: String, + }, + Finished { + outcome: RunOutcome, + reason: String, + exit_code: Option, + }, +} + +impl RunChange { + pub(crate) const fn event_type(&self) -> &'static str { + match self { + Self::Launched { .. } => "run_launched", + Self::LaunchFailed { .. } => "run_launch_failed", + Self::Started => "run_started", + Self::NodeStarted { .. } => "run_node_started", + Self::NodeFinished { .. } => "run_node_finished", + Self::Unresponsive => "run_unresponsive", + Self::Recovered => "run_recovered", + Self::CancelRequested { .. } => "run_cancel_requested", + Self::Finished { .. } => "run_terminated", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RunChangeRequest { + pub event_id: Uuid, + pub run_id: Uuid, + pub claim_id: Option, + pub change: RunChange, +} + +/// Constructed after authentication; this authority is never deserialized from a request body. +#[derive(Clone, Copy, Debug)] +pub enum RunAuthority<'a> { + Scheduler(&'a crate::scheduler::models::SchedulerLease), + Worker { run_id: Uuid, claim_id: Uuid }, + Operator(&'a crate::credentials::models::AuditContext), +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct QueueCandidate { + pub(crate) request_json: RunRequest, + pub(crate) queued_at: DateTime, +} + +/// Fairness covers runnable launcher scopes, not queues the installation cannot dispatch. +#[derive(Clone, Copy, Debug)] +pub struct RunDispatch<'a> { + pub scope: Option<(Uuid, &'a str)>, + pub eligible: Option<&'a BTreeSet<(Uuid, String)>>, +} + +impl<'a> From> for RunDispatch<'a> { + fn from(scope: Option<(Uuid, &'a str)>) -> Self { + Self { + scope, + eligible: None, + } + } +} + +impl<'a> From<(Uuid, &'a str)> for RunDispatch<'a> { + fn from(scope: (Uuid, &'a str)) -> Self { + Some(scope).into() + } +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct ChangingRun { + pub(crate) status: RunStatus, + pub(crate) worker_instance: Option, + pub(crate) cancel_requested_at: Option>, + pub(crate) node_counts_json: BTreeMap, +} + +/// Virtual parents and physical children use the same lifecycle evidence during group progress. +#[derive(Debug, Deserialize)] +pub(crate) struct GroupRun { + pub(crate) request_json: RunRequest, + pub(crate) queued_at: DateTime, + #[serde(flatten)] + pub(crate) state: ChangingRun, +} + +/// Parent overlap locks and installation-wide limits occupy different resource namespaces. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum ConcurrencyResource { + Limit(String), + Overlap(Uuid, String, String), +} + +/// A transaction-local safety calculation, never a second persisted allocation ledger. +#[derive(Debug)] +pub(crate) struct GroupCapacity { + pub(crate) available: BTreeMap, + pub(crate) held: BTreeMap>, + pub(crate) needed: BTreeMap>, + pub(crate) waiting: Vec, +} + +#[derive(Debug)] +pub(crate) struct GroupContext { + pub(crate) parent: GroupRun, + pub(crate) children: Vec, + pub(crate) definitions: crate::declarations::models::Definitions, + pub(crate) critical_failed: bool, +} + +/// Differing retries retain conflict evidence while preserving the first output bytes. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct OutputPublication { + pub inserted: u32, + pub duplicates: u32, + pub conflicts: u32, +} + +/// One invocation's records; the server owns its claim namespace, start time and version. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RecallBatch { + pub invocation_id: String, + pub records: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RecallRecord { + pub table: sqlbuild_recall::schema::models::Table, + pub values: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ClaimMetadata { + pub(crate) claimed_at: DateTime, + pub(crate) sqb_version: String, +} + +/// A physical launcher uses the database clock and current liveness, never an old run event. +#[derive(Debug, Deserialize)] +pub struct RunObservation { + pub status: RunStatus, + pub cancel_requested_at: Option>, + pub claimed_at: Option>, + pub last_heartbeat_at: Option>, + pub observed_at: DateTime, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs new file mode 100644 index 0000000000..7a27408a66 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/calendar.rs @@ -0,0 +1,31 @@ +//! Map local cron times without using the cron dependency's different fold policy. + +use chrono::{DateTime, LocalResult, NaiveDateTime, TimeZone, Utc}; +use chrono_tz::{GapInfo, Tz}; + +use crate::store::errors::ControlError; + +/// During the first fold, smaller wall times in the second fold may still be in the future. +pub(crate) fn search_start(timezone: Tz, after: DateTime) -> NaiveDateTime { + let local = after.with_timezone(&timezone).naive_local(); + match timezone.from_local_datetime(&local) { + LocalResult::Ambiguous(earlier, later) if after < later.to_utc() => { + local - (later.to_utc() - earlier.to_utc()) + } + _ => local, + } +} + +pub(crate) fn resolve_wall_time( + timezone: Tz, + wall_time: NaiveDateTime, +) -> Result, ControlError> { + match timezone.from_local_datetime(&wall_time) { + LocalResult::Single(time) => Ok(time.to_utc()), + LocalResult::Ambiguous(_, later) => Ok(later.to_utc()), + LocalResult::None => GapInfo::new(&wall_time, &timezone) + .and_then(|gap| gap.end) + .map(|time| time.to_utc()) + .ok_or_else(|| ControlError::Invalid("cron time has no representable gap end".into())), + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs new file mode 100644 index 0000000000..ba0d626c09 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_due_triggers.rs @@ -0,0 +1,40 @@ +//! Drain a bounded due-trigger page, renewing the lease between persisted evaluations. + +use uuid::Uuid; + +use crate::scheduler::_helpers::drain_sensors::drain_sensors; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::{SchedulerCycle, SchedulerLease}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn drain_due_triggers( + store: &ControlStore, + mut lease: SchedulerLease, + mut cycle: SchedulerCycle, +) -> Result<(SchedulerLease, SchedulerCycle), ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT s.trigger_id FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE t.kind='schedule' AND s.status='running' AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at, s.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + for row in rows { + let trigger_id: Uuid = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(plan) = evaluate_trigger(store, &lease, trigger_id).await? { + cycle.evaluated += 1; + cycle.ticks += plan.actions.len(); + } + lease = renew_lease(store, &lease).await?; + } + let (lease, sensor_evaluations) = drain_sensors(store, lease).await?; + cycle.evaluated += sensor_evaluations; + Ok((lease, cycle)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs new file mode 100644 index 0000000000..964ee871b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/drain_sensors.rs @@ -0,0 +1,42 @@ +//! The scheduler drains built-in sensors without importing project code. + +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::main::complete_custom_sensor::complete_custom_sensor; +use crate::sensors::models::SensorCondition; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub(crate) async fn drain_sensors( + store: &ControlStore, + mut lease: SchedulerLease, +) -> Result<(SchedulerLease, usize), ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id WHERE t.kind='sensor' AND s.status='running' \ + AND s.next_due_at<=$1 AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL \ + ORDER BY s.next_due_at,t.trigger_id LIMIT 100", vec![Parameter::Time(now)]).await?; + commit(transaction).await?; + let mut evaluated = 0; + for row in rows { + let trigger_id = serde_json::from_value(row["trigger_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(request) = begin_sensor_evaluation(store, &lease, trigger_id).await? { + if matches!(request.definition.when, SensorCondition::Custom { .. }) { + complete_custom_sensor(store, &lease, &request).await?; + } else { + complete_builtin_sensor(store, &lease, &request).await?; + } + evaluated += 1; + } + lease = renew_lease(store, &lease).await?; + } + Ok((lease, evaluated)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs new file mode 100644 index 0000000000..75337f66d1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/expansion.rs @@ -0,0 +1,58 @@ +//! Bound cron backlog work without losing omitted times or requesting an obsolete latest run. + +use std::ops::Bound::{Excluded, Unbounded}; + +use chrono::{DateTime, Utc}; + +use crate::declarations::models::{Schedule, TriggerDefinition}; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::_next_cron_time::next_cron_time; +use crate::scheduler::main::_previous_cron_time::previous_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::models::CatchUp; +use crate::scheduler::types::ScheduleExpansion; +use crate::store::errors::ControlError; + +pub(crate) fn expand_schedule( + trigger: &TriggerDefinition, + from: DateTime, + now: DateTime, +) -> Result { + let (expression, timezone) = match &trigger.schedule { + Schedule::Cron { + expression, + timezone, + } => (expression, timezone), + Schedule::Calendar { times, .. } => { + return Ok(( + times.range((Excluded(from), Unbounded)).copied().collect(), + None, + )); + } + }; + let cron = parse_cron(expression, timezone)?; + let latest = trigger.catch_up == CatchUp::Latest; + let mut due: Vec> = Vec::new(); + let mut cursor = from; + for _ in 0..(MAX_TICKS_PER_BATCH - usize::from(latest)) { + cursor = next_cron_time(&cron, cursor)?; + due.push(cursor); + if cursor > now { + return Ok((due, None)); + } + } + let continuation = next_cron_time(&cron, cursor)?; + if latest && continuation <= now { + let latest = previous_cron_time(&cron, now)?; + due.push(latest); + return Ok(( + due, + Some(if latest == continuation { + next_cron_time(&cron, latest)? + } else { + continuation + }), + )); + } + Ok((due, Some(continuation))) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs new file mode 100644 index 0000000000..c007a30d78 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod calendar; +pub(crate) mod drain_due_triggers; +pub(crate) mod drain_sensors; +pub(crate) mod expansion; +pub(crate) mod persistence; +pub(crate) mod ticks; diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs new file mode 100644 index 0000000000..0f659487c0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -0,0 +1,75 @@ +//! Deterministic tick insertion gates all run facts; retries cannot queue a second run. + +use serde_json::json; + +use crate::runs::main::job_write_set::job_write_set; +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::scheduler::models::{DueTrigger, SchedulerLease, TickDecision, TickPlan}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn persist_ticks( + transaction: ControlTransaction, + (trigger, lease): (&DueTrigger, &SchedulerLease), + plan: &TickPlan, +) -> Result { + let now = transaction.now; + let (mut transaction, write_set) = job_write_set( + transaction, + trigger.snapshot_id, + &trigger.job_json.selection, + ) + .await?; + for action in &plan.actions { + let (outcome, run, covered) = match action.decision { + TickDecision::Requested { run_id } => ("requested", Some(run_id), None), + TickDecision::Covered { covered_by_run_id } => { + ("covered", None, Some(covered_by_run_id)) + } + }; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id, details_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (trigger_id, scheduled_for) DO NOTHING RETURNING tick_id", + vec![Parameter::Id(action.tick_id), Parameter::Id(trigger.trigger_id), Parameter::Time(action.scheduled_for), Parameter::Time(now), Parameter::Integer(lease.epoch), + Parameter::Text(outcome.into()), Parameter::OptionalId(run), Parameter::OptionalId(covered), Parameter::Json(json!({}))]).await?; + if let Some(run_id) = run + && !inserted.is_empty() + { + let request = RunRequest { + run_id, + location_id: trigger.location_id, + target: trigger.target.clone(), + snapshot_id: trigger.snapshot_id, + image_id: trigger.image_id.clone(), + kind: trigger.run_kind, + origin: RunOrigin::Schedule, + job_key: Some(trigger.job_key.clone()), + parent_run_id: None, + root_run_id: run_id, + trigger_id: Some(trigger.trigger_id), + tick_id: Some(action.tick_id), + rerun_of_run_id: None, + requested_by: "system:scheduler".into(), + priority: 0, + selection: trigger.job_json.selection.clone(), + write_set: if trigger.run_kind == RunKind::Group { + Default::default() + } else { + write_set.clone() + }, + settings: trigger.job_json.settings.clone(), + window: None, + concurrency_groups: trigger.job_json.concurrency_groups.clone(), + overlap: trigger.job_json.overlap, + }; + transaction = queue_run(transaction, &request).await?; + } + } + (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET last_tick_for=$1, next_due_at=$2, updated_at=$3 WHERE trigger_id=$4", + vec![Parameter::OptionalTime(plan.last_tick_for), Parameter::OptionalTime(plan.next_due_at), Parameter::Time(now), Parameter::Id(trigger.trigger_id)]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs new file mode 100644 index 0000000000..a4de72e3d6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs @@ -0,0 +1,68 @@ +//! Stable IDs and dispositions, shared by catch-up and ordinary schedule evaluation. + +use chrono::{DateTime, SecondsFormat, Utc}; +use uuid::Uuid; + +use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickDecision, TickState}; + +pub(crate) fn catch_up_actions(state: &TickState, due: &[DateTime]) -> Vec { + let covering_run = (state.overlap == Overlap::Coalesce) + .then_some(state.waiting_run) + .flatten(); + match (covering_run, state.catch_up) { + (Some(run_id), _) => due + .iter() + .map(|time| action(state.trigger_id, *time, Some(run_id))) + .collect(), + (None, CatchUp::All { max_runs }) => due + .iter() + .take(max_runs.get()) + .map(|time| action(state.trigger_id, *time, None)) + .collect(), + (None, CatchUp::Latest) => { + let latest = due.last().copied(); + let run_id = latest.map(|time| requested_run(tick_id(state.trigger_id, time))); + due.iter() + .map(|time| { + action( + state.trigger_id, + *time, + (Some(*time) != latest).then_some(run_id).flatten(), + ) + }) + .collect() + } + } +} + +pub(crate) fn tick_id(trigger_id: Uuid, scheduled_for: DateTime) -> Uuid { + Uuid::new_v5( + &trigger_id, + scheduled_for + .to_rfc3339_opts(SecondsFormat::Nanos, true) + .as_bytes(), + ) +} + +pub(crate) fn requested_run(tick_id: Uuid) -> Uuid { + Uuid::new_v5(&tick_id, b"run") +} + +pub(crate) fn action( + trigger_id: Uuid, + scheduled_for: DateTime, + covering_run: Option, +) -> TickAction { + let tick_id = tick_id(trigger_id, scheduled_for); + let decision = match covering_run { + Some(covered_by_run_id) => TickDecision::Covered { covered_by_run_id }, + None => TickDecision::Requested { + run_id: requested_run(tick_id), + }, + }; + TickAction { + tick_id, + scheduled_for, + decision, + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/constants.rs b/crates/sqlbuild-orchestration/src/scheduler/constants.rs new file mode 100644 index 0000000000..b6d1c0ff0f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/constants.rs @@ -0,0 +1,8 @@ +//! Leadership timing is shared by embedded and standalone schedulers. + +/// Renew well before another process may take over a stopped scheduler. +pub const RENEW_INTERVAL_SECONDS: u64 = 5; +/// A stopped holder cannot delay failover for more than this database-clock interval. +pub const LEASE_TTL_SECONDS: i64 = 30; +/// Bound each decision batch, including covered ticks, before opening a write transaction. +pub const MAX_TICKS_PER_BATCH: usize = 10_000; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs new file mode 100644 index 0000000000..20b4286e1b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_acquire_lease.rs @@ -0,0 +1,37 @@ +//! Elect one scheduler under the control store's serialized writer lock. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::scheduler::constants::LEASE_TTL_SECONDS; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// An existing live lease is never stolen, including by a duplicate caller with the same holder. +pub(crate) async fn acquire_lease( + store: &ControlStore, + holder: Uuid, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Write).await?; + let now = transaction.now; + let expires_at = now + TimeDelta::seconds(LEASE_TTL_SECONDS); + let (transaction, rows) = query(transaction, + "INSERT INTO sqb_scheduler_lease (name, holder, epoch, expires_at) VALUES ('scheduler',$1,1,$2) \ + ON CONFLICT (name) DO UPDATE SET holder=excluded.holder, epoch=sqb_scheduler_lease.epoch+1, expires_at=excluded.expires_at \ + WHERE sqb_scheduler_lease.expires_at<=$3 RETURNING epoch", + vec![Parameter::Id(holder), Parameter::Time(expires_at), Parameter::Time(now)]).await?; + let lease = rows + .first() + .and_then(|row| row["epoch"].as_i64()) + .map(|epoch| SchedulerLease { + holder, + epoch, + expires_at, + }); + commit(transaction).await?; + Ok(lease) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs new file mode 100644 index 0000000000..2646e237ee --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs @@ -0,0 +1,81 @@ +//! Fence, decide and publish using one serialized transaction and its database clock. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::_helpers::expansion::expand_schedule; +use crate::scheduler::_helpers::persistence::persist_ticks; +use crate::scheduler::main::_plan_ticks::plan_ticks; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::{DueTrigger, SchedulerLease, TickPlan, TickState}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// A removed or paused trigger is inert; a lost fence rolls back every write, including Recall. +pub(crate) async fn evaluate_trigger( + store: &ControlStore, + lease: &SchedulerLease, + trigger_id: Uuid, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id, t.location_id, t.target, t.job_key, t.snapshot_id, s.image_id, t.definition_json, j.definition_json AS job_json, j.kind AS run_kind, \ + state.started_at, state.last_tick_for, state.status FROM sqb_triggers t \ + JOIN sqb_trigger_state state ON state.trigger_id=t.trigger_id JOIN sqb_snapshots s ON s.snapshot_id=t.snapshot_id \ + JOIN sqb_jobs j ON j.location_id=t.location_id AND j.target=t.target AND j.job_key=t.job_key \ + WHERE t.trigger_id=$1 AND t.kind='schedule' AND t.removed_in_snapshot_id IS NULL", + vec![Parameter::Id(trigger_id)]).await?; + let Some(trigger) = rows + .into_iter() + .next() + .map(decode_row::) + .transpose()? + else { + return Ok(None); + }; + if trigger.status != TriggerStatus::Running { + return Ok(None); + } + let now = transaction.now; + let from = trigger.last_tick_for.unwrap_or( + trigger + .started_at + .checked_sub_signed(TimeDelta::microseconds(1)) + .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, + ); + let (due_times, continuation) = expand_schedule(&trigger.definition_json, from, now)?; + let (transaction, rows) = query(transaction, + "SELECT r.run_id FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + AND r.status='queued' AND r.cancel_requested_at IS NULL \ + AND NOT EXISTS (SELECT 1 FROM sqb_run_liveness l WHERE l.run_id=r.run_id) ORDER BY r.queued_at, r.run_id LIMIT 1", + vec![Parameter::Id(trigger.location_id), Parameter::Text(trigger.target.clone()), Parameter::Text(trigger.job_key.clone())]).await?; + let waiting_run = rows + .first() + .and_then(|row| row.get("run_id")) + .and_then(serde_json::Value::as_str) + .map(Uuid::parse_str) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let plan = plan_ticks( + &TickState { + trigger_id, + enabled: true, + last_tick_for: trigger.last_tick_for, + due_times, + continuation, + catch_up: trigger.definition_json.catch_up, + overlap: trigger.job_json.overlap, + waiting_run, + }, + now, + )?; + let transaction = persist_ticks(transaction, (&trigger, lease), &plan).await?; + let transaction = check_lease(transaction, lease).await?; + commit(transaction).await?; + Ok(Some(plan)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs new file mode 100644 index 0000000000..90c51039cc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_next_cron_time.rs @@ -0,0 +1,28 @@ +//! Calculate the first distinct UTC tick after a persisted cursor. + +use chrono::{DateTime, Utc}; + +use crate::scheduler::_helpers::calendar::{resolve_wall_time, search_start}; +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Ambiguities select the later fold; missing times move to the end of the actual IANA gap. +pub(in crate::scheduler) fn next_cron_time( + schedule: &CronSchedule, + after: DateTime, +) -> Result, ControlError> { + let mut cursor = search_start(schedule.timezone, after).and_utc(); + loop { + let candidate = schedule + .expression + .find_next_occurrence(&cursor, false) + .map_err(|error| { + ControlError::Invalid(format!("cron has no next occurrence: {error}")) + })?; + let time = resolve_wall_time(schedule.timezone, candidate.naive_utc())?; + if time > after { + return Ok(time); + } + cursor = candidate; + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs new file mode 100644 index 0000000000..842e9308eb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs @@ -0,0 +1,61 @@ +//! Pure scheduling decisions: crashes and clock jumps cannot change a tick's identity. + +use std::collections::BTreeSet; + +use chrono::{DateTime, Utc}; + +use crate::scheduler::_helpers::ticks::catch_up_actions; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::models::{TickPlan, TickState}; +use crate::store::errors::ControlError; + +/// Return facts to append and the cursor to commit atomically with them. +pub(in crate::scheduler) fn plan_ticks( + state: &TickState, + now: DateTime, +) -> Result { + if !state.enabled { + return Ok(TickPlan { + actions: vec![], + last_tick_for: state.last_tick_for, + next_due_at: None, + }); + } + if state.due_times.len() > MAX_TICKS_PER_BATCH { + return Err(ControlError::Invalid( + "schedule expansion exceeds the bounded tick batch".into(), + )); + } + let pending: BTreeSet> = state + .due_times + .iter() + .copied() + .filter(|time| state.last_tick_for.is_none_or(|last| *time > last)) + .collect(); + let due: Vec> = pending + .iter() + .copied() + .take_while(|time| *time <= now) + .collect(); + let actions = catch_up_actions(state, &due); + let last_tick_for = actions + .iter() + .rev() + .find(|action| { + state + .continuation + .is_none_or(|next| action.scheduled_for < next) + }) + .map(|action| action.scheduled_for) + .or(state.last_tick_for); + let next_due_at = pending + .into_iter() + .chain(state.continuation) + .filter(|time| last_tick_for.is_none_or(|last| *time > last)) + .min(); + Ok(TickPlan { + actions, + last_tick_for, + next_due_at, + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs new file mode 100644 index 0000000000..3297e839ba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_previous_cron_time.rs @@ -0,0 +1,30 @@ +//! Locate the latest due tick without expanding an unbounded catch-up backlog. + +use chrono::{DateTime, Utc}; + +use crate::scheduler::_helpers::calendar::resolve_wall_time; +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Fold and gap mapping is identical to forward expansion, including the later ambiguous instant. +pub(in crate::scheduler) fn previous_cron_time( + schedule: &CronSchedule, + at: DateTime, +) -> Result, ControlError> { + let mut cursor = at.with_timezone(&schedule.timezone).naive_local().and_utc(); + let mut inclusive = true; + loop { + let candidate = schedule + .expression + .find_previous_occurrence(&cursor, inclusive) + .map_err(|error| { + ControlError::Invalid(format!("cron has no previous occurrence: {error}")) + })?; + let time = resolve_wall_time(schedule.timezone, candidate.naive_utc())?; + if time <= at { + return Ok(time); + } + cursor = candidate; + inclusive = false; + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs new file mode 100644 index 0000000000..1d94e6944a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_scheduler_cycle.rs @@ -0,0 +1,65 @@ +//! A cycle renews leadership and drains a bounded set of persisted due triggers. + +use std::time::Duration; + +use chrono::{DateTime, Utc}; + +use crate::runs::constants::MONITOR_BATCH_SIZE; +use crate::runs::main::monitor_runs::monitor_runs; +use crate::scheduler::_helpers::drain_due_triggers::drain_due_triggers; +use crate::scheduler::constants::RENEW_INTERVAL_SECONDS; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::{SchedulerCycle, SchedulerState}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) async fn scheduler_cycle( + store: &ControlStore, + mut state: SchedulerState, +) -> Result<(SchedulerState, SchedulerCycle), ControlError> { + let renewal = match state.lease.as_ref() { + Some(lease) => renew_lease(store, lease).await.map(Some), + None => acquire_lease(store, state.holder).await, + }; + state.lease = match renewal { + Ok(lease) => lease, + Err(ControlError::Fenced) => None, + Err(error) => return Err(error), + }; + let cycle = SchedulerCycle { + leader: state.lease.is_some(), + evaluated: 0, + ticks: 0, + unresponsive: 0, + wait_for: Duration::from_secs(RENEW_INTERVAL_SECONDS), + }; + let Some(lease) = state.lease.clone() else { + return Ok((state, cycle)); + }; + let (lease, mut cycle) = drain_due_triggers(store, lease, cycle).await?; + cycle.unresponsive = monitor_runs(store, &lease).await?; + if cycle.unresponsive == MONITOR_BATCH_SIZE { + cycle.wait_for = Duration::ZERO; + } + state.lease = Some(lease); + let transaction = begin(store, TransactionMode::Read).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT MIN(s.next_due_at) AS next_due_at FROM sqb_trigger_state s JOIN sqb_triggers t ON t.trigger_id=s.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id \ + WHERE s.status='running' AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", vec![]).await?; + let next: Option> = serde_json::from_value(rows[0]["next_due_at"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if let Some(next) = next { + cycle.wait_for = (next - now) + .to_std() + .unwrap_or(Duration::ZERO) + .min(cycle.wait_for); + } + commit(transaction).await?; + Ok((state, cycle)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs new file mode 100644 index 0000000000..4c9d7a9247 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/check_lease.rs @@ -0,0 +1,22 @@ +//! Shared fence validation using a fresh database-clock reading in the writer transaction. + +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::main::refresh_clock::refresh_clock; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn check_lease( + transaction: ControlTransaction, + lease: &SchedulerLease, +) -> Result { + let transaction = refresh_clock(transaction).await?; + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT epoch FROM sqb_scheduler_lease WHERE name='scheduler' AND holder=$1 AND epoch=$2 AND expires_at>$3", + vec![Parameter::Id(lease.holder), Parameter::Integer(lease.epoch), Parameter::Time(now)]).await?; + if rows.is_empty() { + return Err(ControlError::Fenced); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs new file mode 100644 index 0000000000..0ffa73f861 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs @@ -0,0 +1,29 @@ +//! Tick history contains immutable evaluations, including covered schedule times. + +use uuid::Uuid; + +use crate::scheduler::models::TickRecord; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +pub async fn list_ticks( + store: &ControlStore, + trigger_id: Uuid, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query(transaction, + "SELECT tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id, reason \ + FROM sqb_ticks WHERE trigger_id=$1 ORDER BY scheduled_for DESC LIMIT $2", + vec![Parameter::Id(trigger_id), Parameter::Integer(i64::from(limit.min(1000)))]).await?; + let ticks = rows + .into_iter() + .map(decode_row) + .collect::, _>>()?; + commit(transaction).await?; + Ok(ticks) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs new file mode 100644 index 0000000000..f91c3d1180 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs @@ -0,0 +1,12 @@ +pub(in crate::scheduler) mod _acquire_lease; +pub(in crate::scheduler) mod _evaluate_trigger; +pub(in crate::scheduler) mod _next_cron_time; +pub(in crate::scheduler) mod _plan_ticks; +pub(in crate::scheduler) mod _previous_cron_time; +pub(in crate::scheduler) mod _scheduler_cycle; +pub(crate) mod check_lease; +pub mod list_ticks; +pub(crate) mod next_schedule_time; +pub mod parse_cron; +pub mod renew_lease; +pub mod run_scheduler; diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs new file mode 100644 index 0000000000..880e4ad7c9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/next_schedule_time.rs @@ -0,0 +1,28 @@ +//! Find the next due time for a resolved cron or calendar declaration. + +use std::ops::Bound::{Excluded, Unbounded}; + +use chrono::{DateTime, Utc}; + +use crate::declarations::models::Schedule; +use crate::scheduler::main::_next_cron_time::next_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::store::errors::ControlError; + +pub(crate) fn next_schedule_time( + schedule: &Schedule, + after: DateTime, +) -> Result>, ControlError> { + match schedule { + Schedule::Cron { + expression, + timezone, + } => Ok(Some(next_cron_time( + &parse_cron(expression, timezone)?, + after, + )?)), + Schedule::Calendar { times, .. } => { + Ok(times.range((Excluded(after), Unbounded)).next().copied()) + } + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs new file mode 100644 index 0000000000..50dbeee4b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/parse_cron.rs @@ -0,0 +1,24 @@ +//! Validate POSIX minute-resolution schedules and IANA time zones before publication. + +use croner::parser::{CronParser, Seconds, Year}; + +use crate::scheduler::models::CronSchedule; +use crate::store::errors::ControlError; + +/// Day-of-month and day-of-week use POSIX OR, not Quartz AND semantics. +pub fn parse_cron(expression: &str, timezone: &str) -> Result { + let expression = CronParser::builder() + .seconds(Seconds::Disallowed) + .year(Year::Disallowed) + .dom_and_dow(false) + .build() + .parse(expression) + .map_err(|error| ControlError::Invalid(format!("invalid cron expression: {error}")))?; + let timezone = timezone + .parse() + .map_err(|error| ControlError::Invalid(format!("invalid IANA time zone: {error}")))?; + Ok(CronSchedule { + expression, + timezone, + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs b/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs new file mode 100644 index 0000000000..5e9811a5a0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/renew_lease.rs @@ -0,0 +1,30 @@ +//! A paused or superseded scheduler cannot renew an expired fence. + +use chrono::TimeDelta; + +use crate::scheduler::constants::LEASE_TTL_SECONDS; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Renewal preserves the epoch; only takeover allocates a new epoch. +pub async fn renew_lease( + store: &ControlStore, + lease: &SchedulerLease, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let transaction = check_lease(transaction, lease).await?; + let expires_at = transaction.now + TimeDelta::seconds(LEASE_TTL_SECONDS); + let (transaction, _) = query(transaction, + "UPDATE sqb_scheduler_lease SET expires_at=$1 WHERE name='scheduler' AND holder=$2 AND epoch=$3", + vec![Parameter::Time(expires_at), Parameter::Id(lease.holder), Parameter::Integer(lease.epoch)]).await?; + commit(transaction).await?; + Ok(SchedulerLease { + expires_at, + ..lease.clone() + }) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs new file mode 100644 index 0000000000..71954a8f71 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/run_scheduler.rs @@ -0,0 +1,36 @@ +//! Embedded and standalone roles share the same interruptible, durable scheduling loop. + +use tokio::sync::watch; + +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::types::RunLauncher; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Notifications shorten the wait; durable database state, not receipt of a notification, drives work. +pub async fn run_scheduler( + store: &ControlStore, + mut state: SchedulerState, + (mut shutdown, mut changes): (watch::Receiver, watch::Receiver<()>), + mut launcher: impl RunLauncher, +) -> Result<(), ControlError> { + let mut last_lease = None; + let result = async { + while !*shutdown.borrow() { + let (next_state, cycle) = scheduler_cycle(store, state).await?; + state = next_state; + last_lease = state.lease.clone(); + launcher.poll(store, state.lease.as_ref()).await?; + tokio::select! { + _ = tokio::time::sleep(cycle.wait_for.min(std::time::Duration::from_secs(1))) => {}, + result = changes.changed() => { if result.is_err() { break; } }, + result = shutdown.changed() => { if result.is_err() { break; } }, + } + } + Ok(()) + } + .await; + let stopped = launcher.shutdown(store, last_lease.as_ref()).await; + result.and(stopped) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/mod.rs new file mode 100644 index 0000000000..f5b66b8aca --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/mod.rs @@ -0,0 +1,10 @@ +//! Fenced scheduler leadership and deterministic decisions over durable due times. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs new file mode 100644 index 0000000000..39594ed147 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -0,0 +1,155 @@ +//! Values owned by the scheduler, independent of Python and runtime timers. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::{JobDefinition, TriggerDefinition, TriggerStatus}; + +/// A holder must present the exact epoch for every scheduler write. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SchedulerLease { + pub holder: Uuid, + pub epoch: i64, + pub expires_at: DateTime, +} + +/// Process identity survives cycles; the database remains authoritative for leadership. +#[derive(Clone, Debug)] +pub struct SchedulerState { + pub(crate) holder: Uuid, + pub(crate) lease: Option, +} + +impl SchedulerState { + #[must_use] + pub const fn new(holder: Uuid) -> Self { + Self { + holder, + lease: None, + } + } +} + +#[derive(Clone, Debug)] +pub struct SchedulerCycle { + pub leader: bool, + pub evaluated: usize, + pub ticks: usize, + pub unresponsive: usize, + pub wait_for: std::time::Duration, +} + +/// Validated at the native boundary and evaluated as wall-clock times with SQLBuild's DST policy. +#[derive(Clone, Debug)] +pub struct CronSchedule { + pub(crate) expression: croner::Cron, + pub(crate) timezone: chrono_tz::Tz, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum CatchUp { + #[default] + Latest, + /// Process the oldest bounded batch; later due times remain due for the next transaction. + All { + #[schema(value_type = usize, minimum = 1)] + max_runs: std::num::NonZeroUsize, + }, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Overlap { + #[default] + Coalesce, + Queue, + Allow, +} + +impl Overlap { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Coalesce => "coalesce", + Self::Queue => "queue", + Self::Allow => "allow", + } + } +} + +/// Due times come from the persisted calendar or cron expansion, never elapsed sleep time. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TickState { + pub trigger_id: Uuid, + pub enabled: bool, + pub last_tick_for: Option>, + pub due_times: Vec>, + /// First omitted time in a bounded expansion; never advance the cursor past an unrecorded tick. + pub continuation: Option>, + pub catch_up: CatchUp, + pub overlap: Overlap, + /// Only unclaimed waiting work can cover a later tick; executing work cannot cover future data. + pub waiting_run: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum TickDecision { + Requested { run_id: Uuid }, + Covered { covered_by_run_id: Uuid }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TickAction { + pub tick_id: Uuid, + pub scheduled_for: DateTime, + pub decision: TickDecision, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TickPlan { + pub actions: Vec, + pub last_tick_for: Option>, + pub next_due_at: Option>, +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DueTrigger { + pub(crate) trigger_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) job_key: String, + pub(crate) snapshot_id: Uuid, + pub(crate) image_id: Option, + pub(crate) definition_json: TriggerDefinition, + pub(crate) job_json: JobDefinition, + pub(crate) run_kind: crate::runs::models::RunKind, + pub(crate) started_at: DateTime, + pub(crate) last_tick_for: Option>, + pub(crate) status: TriggerStatus, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum TickOutcome { + Requested, + Skipped, + Covered, + Failed, + Deferred, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct TickRecord { + pub tick_id: Uuid, + pub trigger_id: Uuid, + pub scheduled_for: DateTime, + pub evaluated_at: DateTime, + pub epoch: i64, + pub outcome: TickOutcome, + pub run_id: Option, + pub covered_by_run_id: Option, + pub reason: Option, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests.rs b/crates/sqlbuild-orchestration/src/scheduler/tests.rs new file mode 100644 index 0000000000..3a3124f551 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests.rs @@ -0,0 +1,16 @@ +#[path = "../../tests/scheduler/test_contracts.rs"] +pub(crate) mod contracts; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_backlog.rs"] +mod test_backlog; +#[path = "tests/test_cron.rs"] +mod test_cron; +#[path = "../../tests/scheduler/test_sensor_fairness.rs"] +mod test_sensor_fairness; +#[path = "tests/test_simulation.rs"] +mod test_simulation; +#[path = "tests/test_ticks.rs"] +mod test_ticks; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs new file mode 100644 index 0000000000..0484e35cac --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -0,0 +1,129 @@ +//! Clock-independent schedule fixtures. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use proptest::prelude::any; +use proptest::test_runner::{Config, TestCaseError, TestCaseResult, TestRunner}; +use proptest::{prop_assert, prop_assert_eq}; +use uuid::Uuid; + +use crate::scheduler::main::_plan_ticks::plan_ticks; +use crate::scheduler::models::{CatchUp, Overlap, TickAction, TickDecision, TickPlan, TickState}; +use crate::scheduler::tests::test_types::TickTestCase; + +pub(super) fn time(second: i64) -> DateTime { + DateTime::from_timestamp(second, 0).expect("synthetic schedule time is representable") +} + +pub(super) fn state(test_case: &TickTestCase) -> TickState { + TickState { + trigger_id: Uuid::from_u128(19), + enabled: test_case.enabled, + last_tick_for: test_case.last_second.map(time), + due_times: test_case + .due_seconds + .iter() + .map(|second| time(*second)) + .collect(), + catch_up: test_case.catch_up, + overlap: test_case.overlap, + waiting_run: test_case.active.then_some(Uuid::from_u128(31)), + continuation: None, + } +} + +pub(super) fn request_seconds(plan: &TickPlan) -> Vec { + plan.actions + .iter() + .filter(|action| matches!(action.decision, TickDecision::Requested { .. })) + .map(|action| action.scheduled_for.timestamp()) + .collect() +} + +pub(super) fn covered_seconds(plan: &TickPlan) -> Vec { + plan.actions + .iter() + .filter(|action| matches!(action.decision, TickDecision::Covered { .. })) + .map(|action| action.scheduled_for.timestamp()) + .collect() +} + +fn exercise_history(due: Vec, history: Vec<(i64, bool)>, catch_up: CatchUp) -> TestCaseResult { + let expected: BTreeSet = due.iter().copied().collect(); + let mut state = TickState { + trigger_id: Uuid::from_u128(19), + enabled: true, + last_tick_for: None, + due_times: due.into_iter().map(time).collect(), + catch_up, + overlap: Overlap::Queue, + waiting_run: None, + continuation: None, + }; + let mut committed: BTreeMap = BTreeMap::new(); + for (clock, commit_succeeds) in history { + let plan = plan_ticks(&state, time(clock)) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + let Some(plan) = commit_succeeds.then_some(plan) else { + continue; + }; + for action in plan.actions { + prop_assert!(action.scheduled_for.timestamp() <= clock); + prop_assert!( + committed + .insert(action.scheduled_for.timestamp(), action) + .is_none(), + "a committed tick was emitted twice" + ); + } + state.last_tick_for = plan.last_tick_for; + } + for _ in 0..=expected.len() { + let plan = plan_ticks(&state, time(1001)) + .map_err(|error| TestCaseError::fail(error.to_string()))?; + let Some(_) = plan.actions.first() else { + break; + }; + for action in plan.actions { + prop_assert!( + committed + .insert(action.scheduled_for.timestamp(), action) + .is_none(), + "restart repeated a committed tick" + ); + } + state.last_tick_for = plan.last_tick_for; + } + prop_assert_eq!( + committed.keys().copied().collect::>(), + expected, + "a due tick was lost or invented" + ); + prop_assert_eq!( + committed + .values() + .map(|action| action.tick_id) + .collect::>() + .len(), + committed.len() + ); + Ok(()) +} + +pub(super) fn simulate(catch_up: CatchUp) -> Result<(), String> { + let strategy = ( + proptest::collection::vec(0_i64..1000, 0..128), + proptest::collection::vec((-100_i64..1100, any::()), 0..128), + ); + let mut property_runner = TestRunner::new(Config { + cases: 512, + failure_persistence: None, + ..Config::default() + }); + property_runner + .run(&strategy, |(due, history)| { + exercise_history(due, history, catch_up) + }) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs new file mode 100644 index 0000000000..ac9985f292 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_backlog.rs @@ -0,0 +1,107 @@ +use std::collections::BTreeSet; +use std::num::NonZeroUsize; +use std::time::{Duration, Instant}; + +use crate::declarations::models::{Schedule, TriggerDefinition, TriggerStatus}; +use crate::scheduler::_helpers::expansion::expand_schedule; +use crate::scheduler::constants::MAX_TICKS_PER_BATCH; +use crate::scheduler::main::_plan_ticks::plan_ticks; +use crate::scheduler::models::{CatchUp, Overlap, TickDecision, TickState}; +use crate::scheduler::tests::helpers::time; +use crate::scheduler::tests::test_types::BacklogTestCase; +use uuid::Uuid; + +#[test] +fn given_a_backlog_larger_than_two_batches_when_drained_then_latest_is_current_and_every_tick_is_retained() + { + let test_cases = [ + BacklogTestCase { + description: "latest records covered prefixes without requesting an obsolete run", + catch_up: CatchUp::Latest, + expected_latest: 1_200_180, + expected_count: 20_003, + }, + BacklogTestCase { + description: "all drains every requested tick in bounded batches", + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(MAX_TICKS_PER_BATCH).expect("nonzero bound"), + }, + expected_latest: 1_200_180, + expected_count: 20_003, + }, + ]; + for test_case in test_cases { + let started = Instant::now(); + let trigger = TriggerDefinition { + description: "orders backlog".into(), + job_key: "orders".into(), + starts_at: Some(time(60)), + default_status: TriggerStatus::Running, + catch_up: test_case.catch_up, + schedule: Schedule::Cron { + expression: "* * * * *".into(), + timezone: "UTC".into(), + }, + }; + let mut cursor = None; + let mut recorded: BTreeSet = BTreeSet::new(); + let mut requested: BTreeSet = BTreeSet::new(); + for _ in 0..4 { + let (due_times, continuation) = expand_schedule( + &trigger, + cursor.unwrap_or(time(0)), + time(test_case.expected_latest), + ) + .expect("bounded expansion"); + let plan = plan_ticks( + &TickState { + trigger_id: Uuid::from_u128(19), + enabled: true, + last_tick_for: cursor, + due_times, + continuation, + catch_up: test_case.catch_up, + overlap: Overlap::Queue, + waiting_run: None, + }, + time(test_case.expected_latest), + ) + .expect("bounded plan"); + for action in plan.actions { + recorded.insert(action.scheduled_for.timestamp()); + let TickDecision::Requested { .. } = action.decision else { + continue; + }; + requested.insert(action.scheduled_for.timestamp()); + } + cursor = plan.last_tick_for; + } + assert_eq!( + recorded, + (1..=test_case.expected_count) + .map(|minute| minute as i64 * 60) + .collect(), + "{}", + test_case.description + ); + assert_eq!( + requested.last().copied(), + Some(test_case.expected_latest), + "{}", + test_case.description + ); + let expected_requests = + [test_case.expected_count, 1][usize::from(test_case.catch_up == CatchUp::Latest)]; + assert_eq!( + requested.len(), + expected_requests, + "{}", + test_case.description + ); + assert!( + started.elapsed() < Duration::from_secs(5), + "bounded cron expansion exceeded five seconds: {}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs new file mode 100644 index 0000000000..fac0537e06 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_cron.rs @@ -0,0 +1,158 @@ +use chrono::{DateTime, SecondsFormat}; + +use crate::scheduler::main::_next_cron_time::next_cron_time; +use crate::scheduler::main::_previous_cron_time::previous_cron_time; +use crate::scheduler::main::parse_cron::parse_cron; +use crate::scheduler::tests::test_types::{CronTestCase, PreviousCronTestCase}; + +#[test] +fn given_cron_and_iana_time_zone_when_crossing_dst_then_utc_ticks_follow_the_explicit_policy() { + let test_cases = [ + CronTestCase { + description: "spring gap ends at 03:00, not 03:30", + expression: "30 2 * * *", + timezone: "America/New_York", + after: "2026-03-08T00:00:00Z", + expected_next: "2026-03-08T07:00:00Z", + }, + CronTestCase { + description: "ambiguous daily time uses the later fold", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T00:00:00Z", + expected_next: "2026-11-01T06:30:00Z", + }, + CronTestCase { + description: "restarting in the first fold does not skip a smaller future wall time", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T05:45:00Z", + expected_next: "2026-11-01T06:30:00Z", + }, + CronTestCase { + description: "a persisted later-fold cursor does not repeat the tick", + expression: "30 1 * * *", + timezone: "America/New_York", + after: "2026-11-01T06:30:00Z", + expected_next: "2026-11-02T06:30:00Z", + }, + CronTestCase { + description: "wildcard schedules also select only the later fold", + expression: "* * * * *", + timezone: "America/New_York", + after: "2026-11-01T05:45:00Z", + expected_next: "2026-11-01T06:00:00Z", + }, + CronTestCase { + description: "half-hour spring gap moves to the actual gap end", + expression: "15 2 * * *", + timezone: "Australia/Lord_Howe", + after: "2026-10-03T13:00:00Z", + expected_next: "2026-10-03T15:30:00Z", + }, + CronTestCase { + description: "half-hour autumn fold uses the later 01:45", + expression: "45 1 * * *", + timezone: "Australia/Lord_Howe", + after: "2026-04-04T13:00:00Z", + expected_next: "2026-04-04T15:15:00Z", + }, + CronTestCase { + description: "an entire skipped date moves to the first instant of the next date", + expression: "0 12 * * *", + timezone: "Pacific/Apia", + after: "2011-12-30T00:00:00Z", + expected_next: "2011-12-30T10:00:00Z", + }, + CronTestCase { + description: "POSIX day-of-month and weekday are alternatives", + expression: "0 9 13 * MON", + timezone: "UTC", + after: "2026-10-10T12:00:00Z", + expected_next: "2026-10-12T09:00:00Z", + }, + CronTestCase { + description: "UTC schedules do not introduce local-zone shifts", + expression: "17 6 * * *", + timezone: "UTC", + after: "2026-03-08T06:17:00Z", + expected_next: "2026-03-09T06:17:00Z", + }, + ]; + for test_case in test_cases { + let schedule = + parse_cron(test_case.expression, test_case.timezone).expect("valid synthetic cron"); + let after = DateTime::parse_from_rfc3339(test_case.after) + .expect("valid start timestamp") + .to_utc(); + let observed = next_cron_time(&schedule, after).expect("next cron time exists"); + assert_eq!( + observed.to_rfc3339_opts(SecondsFormat::Secs, true), + test_case.expected_next, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_latest_tick_search_when_crossing_folds_and_gaps_then_it_agrees_with_forward_tick_policy() +{ + let test_cases = [ + PreviousCronTestCase { + description: "first fold excludes every ambiguous minute", + expression: "* * * * *", + timezone: "America/New_York", + at: "2026-11-01T05:45:00Z", + expected_previous: "2026-11-01T04:59:00Z", + }, + PreviousCronTestCase { + description: "second fold includes the later ambiguous instant", + expression: "* * * * *", + timezone: "America/New_York", + at: "2026-11-01T06:15:00Z", + expected_previous: "2026-11-01T06:15:00Z", + }, + PreviousCronTestCase { + description: "daily missing time fires exactly at the gap end", + expression: "30 2 * * *", + timezone: "America/New_York", + at: "2026-03-08T07:00:00Z", + expected_previous: "2026-03-08T07:00:00Z", + }, + PreviousCronTestCase { + description: "before gap end uses the preceding day", + expression: "30 2 * * *", + timezone: "America/New_York", + at: "2026-03-08T06:59:59Z", + expected_previous: "2026-03-07T07:30:00Z", + }, + PreviousCronTestCase { + description: "half-hour gap resolves to its actual end", + expression: "15 2 * * *", + timezone: "Australia/Lord_Howe", + at: "2026-10-03T15:30:00Z", + expected_previous: "2026-10-03T15:30:00Z", + }, + PreviousCronTestCase { + description: "skipped-date cron fires at the first representable instant", + expression: "0 12 * * *", + timezone: "Pacific/Apia", + at: "2011-12-30T10:00:00Z", + expected_previous: "2011-12-30T10:00:00Z", + }, + ]; + for test_case in test_cases { + let cron = parse_cron(test_case.expression, test_case.timezone).expect("valid cron"); + let at = DateTime::parse_from_rfc3339(test_case.at) + .expect("search time") + .to_utc(); + let previous = previous_cron_time(&cron, at).expect("previous scheduled instant"); + assert_eq!( + previous.to_rfc3339_opts(SecondsFormat::Secs, true), + test_case.expected_previous, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs new file mode 100644 index 0000000000..60e0de2784 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_simulation.rs @@ -0,0 +1,32 @@ +use std::num::NonZeroUsize; + +use crate::scheduler::models::CatchUp; +use crate::scheduler::tests::helpers::simulate; +use crate::scheduler::tests::test_types::SimulationTestCase; + +#[test] +fn given_random_crashes_and_clock_jumps_when_restarting_then_every_due_tick_commits_once() { + let test_cases = [ + SimulationTestCase { + description: "latest catch-up keeps covered tick history through crashes", + catch_up: CatchUp::Latest, + expected_success: true, + }, + SimulationTestCase { + description: "bounded all-catch-up drains rather than loses the backlog", + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(3).expect("nonzero test batch"), + }, + expected_success: true, + }, + ]; + for test_case in test_cases { + let observed = simulate(test_case.catch_up); + assert_eq!( + observed.is_ok(), + test_case.expected_success, + "{}: {observed:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs new file mode 100644 index 0000000000..2aecca4468 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs @@ -0,0 +1,146 @@ +use std::num::NonZeroUsize; + +use crate::scheduler::main::_plan_ticks::plan_ticks; +use crate::scheduler::models::{CatchUp, Overlap}; +use crate::scheduler::tests::helpers::{covered_seconds, request_seconds, state, time}; +use crate::scheduler::tests::test_types::TickTestCase; + +#[test] +fn given_durable_schedule_state_when_time_advances_then_decisions_preserve_due_ticks() { + let test_cases = [ + TickTestCase { + description: "latest catch-up covers earlier ticks and deduplicates calendar entries", + enabled: true, + due_seconds: vec![30, 10, 20, 10, 40], + last_second: None, + now_second: 30, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: false, + expected_requests: vec![30], + expected_covered: vec![10, 20], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "all catch-up leaves the unprocessed backlog due instead of skipping it", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: None, + now_second: 40, + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(2).expect("nonzero test batch"), + }, + overlap: Overlap::Queue, + active: false, + expected_requests: vec![10, 20], + expected_covered: vec![], + expected_last: Some(20), + expected_next: Some(30), + }, + TickTestCase { + description: "coalesced ticks are covered by the already waiting run", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: Some(10), + now_second: 30, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: true, + expected_requests: vec![], + expected_covered: vec![20, 30], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "allow does not coalesce even when a prior run is active", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: Some(10), + now_second: 30, + catch_up: CatchUp::All { + max_runs: NonZeroUsize::new(5).expect("nonzero test batch"), + }, + overlap: Overlap::Allow, + active: true, + expected_requests: vec![20, 30], + expected_covered: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "a backward clock jump cannot move the durable cursor backward", + enabled: true, + due_seconds: vec![10, 20, 30, 40], + last_second: Some(30), + now_second: 20, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: false, + expected_requests: vec![], + expected_covered: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "pause retains the cursor and does not advertise an overdue next tick", + enabled: false, + due_seconds: vec![10, 20, 30], + last_second: Some(10), + now_second: 30, + catch_up: CatchUp::Latest, + overlap: Overlap::Coalesce, + active: false, + expected_requests: vec![], + expected_covered: vec![], + expected_last: Some(10), + expected_next: None, + }, + ]; + for test_case in test_cases { + let input = state(&test_case); + let plan = plan_ticks(&input, time(test_case.now_second)).expect("plan valid schedule"); + assert_eq!( + request_seconds(&plan), + test_case.expected_requests, + "{}", + test_case.description + ); + assert_eq!( + covered_seconds(&plan), + test_case.expected_covered, + "{}", + test_case.description + ); + assert_eq!( + plan.last_tick_for.map(|time| time.timestamp()), + test_case.expected_last + ); + assert_eq!( + plan.next_due_at.map(|time| time.timestamp()), + test_case.expected_next + ); + let retried = plan_ticks(&input, time(test_case.now_second)).expect("retry schedule"); + assert_eq!( + plan, retried, + "a retry has exactly the same deterministic tick and run IDs" + ); + let tick_ids: std::collections::BTreeSet<_> = + plan.actions.iter().map(|action| action.tick_id).collect(); + assert_eq!( + tick_ids.len(), + test_case.expected_requests.len() + test_case.expected_covered.len() + ); + let advanced = crate::scheduler::models::TickState { + last_tick_for: plan.last_tick_for, + ..input + }; + let second = + plan_ticks(&advanced, time(test_case.now_second)).expect("plan from committed cursor"); + assert!(second.actions.iter().all(|next| { + plan.actions + .iter() + .all(|before| before.tick_id != next.tick_id) + })); + } +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs new file mode 100644 index 0000000000..ab3d443e4f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs @@ -0,0 +1,61 @@ +//! Inputs make cursor, catch-up, and overlap decisions observably different. + +use crate::scheduler::models::{CatchUp, Overlap}; +use crate::store::tests::store_test_types::Backend; + +pub(super) struct SensorFairnessTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) pending: usize, + pub(super) builtin_status: &'static str, + pub(super) expected_completed: usize, +} + +#[derive(Debug)] +pub(super) struct TickTestCase { + pub(super) description: &'static str, + pub(super) enabled: bool, + pub(super) due_seconds: Vec, + pub(super) last_second: Option, + pub(super) now_second: i64, + pub(super) catch_up: CatchUp, + pub(super) overlap: Overlap, + pub(super) active: bool, + pub(super) expected_requests: Vec, + pub(super) expected_covered: Vec, + pub(super) expected_last: Option, + pub(super) expected_next: Option, +} + +#[derive(Debug)] +pub(super) struct CronTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) timezone: &'static str, + pub(super) after: &'static str, + pub(super) expected_next: &'static str, +} + +#[derive(Debug)] +pub(super) struct PreviousCronTestCase { + pub(super) description: &'static str, + pub(super) expression: &'static str, + pub(super) timezone: &'static str, + pub(super) at: &'static str, + pub(super) expected_previous: &'static str, +} + +#[derive(Debug)] +pub(super) struct BacklogTestCase { + pub(super) description: &'static str, + pub(super) catch_up: CatchUp, + pub(super) expected_latest: i64, + pub(super) expected_count: usize, +} + +#[derive(Debug)] +pub(super) struct SimulationTestCase { + pub(super) description: &'static str, + pub(super) catch_up: CatchUp, + pub(super) expected_success: bool, +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/types.rs b/crates/sqlbuild-orchestration/src/scheduler/types.rs new file mode 100644 index 0000000000..53e9334d97 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/types.rs @@ -0,0 +1,25 @@ +//! Compound scheduler return values. + +use chrono::{DateTime, Utc}; + +use crate::scheduler::models::SchedulerLease; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Physical process ownership survives leadership loss, but durable writes require a live fence. +pub trait RunLauncher { + fn poll( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> impl Future> + Send; + + fn shutdown( + &mut self, + store: &ControlStore, + lease: Option<&SchedulerLease>, + ) -> impl Future> + Send; +} + +/// Expanded wall-clock times and the first omitted time, if the batch was bounded. +pub(crate) type ScheduleExpansion = (Vec>, Option>); diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs new file mode 100644 index 0000000000..310fde7a9b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs @@ -0,0 +1,65 @@ +//! Indexed reads recover attempts and schedule projections without mutable phase transitions. + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::sensors::models::{DueSensor, SensorEvaluationFact}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_sensor( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT t.trigger_id,t.location_id,t.target,t.snapshot_id,t.definition_json,s.status \ + FROM sqb_triggers t JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + JOIN sqb_locations l ON l.location_id=t.location_id WHERE t.trigger_id=$1 AND t.kind='sensor' \ + AND t.removed_in_snapshot_id IS NULL AND l.archived_at IS NULL", + vec![Parameter::Id(trigger_id)]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} + +pub(crate) async fn last_attempt( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, Option<(SensorEvaluationFact, bool)>), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT e.fact_json, CASE WHEN EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events terminal \ + WHERE terminal.evaluation_id=e.evaluation_id AND terminal.phase<>'requested') THEN 1 ELSE 0 END AS terminal \ + FROM sqb_sensor_evaluation_events e WHERE e.trigger_id=$1 AND e.phase='requested' ORDER BY e.storage_id DESC LIMIT 1", + vec![Parameter::Id(trigger_id)]).await?; + let last = rows + .first() + .map(|row| { + let fact = serde_json::from_value(row["fact_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + Ok::<_, ControlError>((fact, row["terminal"].as_i64() == Some(1))) + }) + .transpose()?; + Ok((transaction, last)) +} + +pub(crate) async fn defer_sensor( + transaction: ControlTransaction, + trigger_id: Uuid, + next: DateTime, +) -> Result { + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1,updated_at=$2 WHERE trigger_id=$3", + vec![ + Parameter::Time(next), + Parameter::Time(now), + Parameter::Id(trigger_id), + ], + ) + .await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs new file mode 100644 index 0000000000..20964ded97 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/completion.rs @@ -0,0 +1,77 @@ +//! Every evaluation result uses the same lease, snapshot, cursor and terminal boundary. + +use chrono::TimeDelta; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::facts::{append_evaluation, commit_cursor}; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn pending_evaluation( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + if evaluation.epoch != lease.epoch { + return Err(ControlError::Fenced); + } + let (transaction, previous) = last_attempt(transaction, evaluation.trigger_id).await?; + let Some((previous, terminal)) = previous else { + return Err(ControlError::Fenced); + }; + if previous.request != *evaluation { + return Err(ControlError::Fenced); + } + if terminal { + commit(transaction).await?; + return Ok(None); + } + let (transaction, sensor) = read_sensor(transaction, evaluation.trigger_id).await?; + let (transaction, cursor) = read_sensor_cursor(transaction, evaluation.trigger_id).await?; + if sensor.is_none_or(|sensor| { + sensor.status != TriggerStatus::Running || sensor.snapshot_id != evaluation.snapshot_id + }) || cursor != evaluation.cursor + { + let (transaction, _) = + append_evaluation(transaction, evaluation, SensorEvaluationOutcome::Abandoned).await?; + commit(transaction).await?; + return Ok(None); + } + Ok(Some(transaction)) +} + +pub(crate) async fn finish_evaluation( + transaction: ControlTransaction, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, + outcome: SensorEvaluationOutcome, +) -> Result<(), ControlError> { + let cursor = match &outcome { + SensorEvaluationOutcome::Completed { cursor, .. } => Some(cursor.clone()), + _ => None, + }; + let (mut transaction, (event_id, storage_id)) = + append_evaluation(transaction, evaluation, outcome).await?; + if let Some(cursor) = cursor { + transaction = commit_cursor( + transaction, + (evaluation.trigger_id, event_id, storage_id), + &cursor, + ) + .await?; + } + let next = evaluation.requested_at + + TimeDelta::seconds(i64::from( + evaluation.definition.minimum_interval_seconds.get(), + )); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs new file mode 100644 index 0000000000..2e4243170e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -0,0 +1,40 @@ +//! Each native condition consumes its bounded page inside the evaluation transaction. + +use uuid::Uuid; + +use crate::sensors::_helpers::nodes::consume_node_events; +use crate::sensors::_helpers::platform::consume_platform_events; +use crate::sensors::_helpers::runs::consume_run_events; +use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn consume_condition( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + match &evaluation.definition.when { + SensorCondition::Custom { .. } => Err(ControlError::Invalid( + "custom sensors require a location result".into(), + )), + SensorCondition::PlatformEvent { kinds } => { + consume_platform_events(transaction, evaluation, kinds).await + } + SensorCondition::NodeEvent { selection, .. } => { + consume_node_events(transaction, evaluation, selection).await + } + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { + consume_run_events( + transaction, + evaluation, + (job_key, kinds), + executed_successfully.as_ref(), + ) + .await + } + } +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs new file mode 100644 index 0000000000..c2e65bb427 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs @@ -0,0 +1,104 @@ +//! Custom callbacks cross only the existing location-scoped outbound-work boundary. + +use std::collections::BTreeSet; + +use chrono::TimeDelta; +use serde_json::json; +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkRequest}; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{ + CustomSensorOutcome, SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn request_evaluation_work( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result { + if !matches!(evaluation.definition.when, SensorCondition::Custom { .. }) { + return Ok(transaction); + } + let request = LocationWorkRequest { + request_id: evaluation.evaluation_id, + location_id: evaluation.location_id, + target: evaluation.target.clone(), + reason: "Evaluate custom sensor".into(), + work: LocationWork::EvaluateSensor { + evaluation: Box::new(evaluation.clone()), + }, + }; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + SELECT $1,$2,$3,snapshot_id,generation,$4,'system:sensor',$5,$6 FROM sqb_snapshots WHERE snapshot_id=$7", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Json(json!(request)), Parameter::Time(evaluation.requested_at), + Parameter::Time(evaluation.requested_at + TimeDelta::minutes(5)), Parameter::Id(evaluation.snapshot_id)]).await?; + Ok(transaction) +} + +pub(crate) async fn read_evaluation_result( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, work) = read_location_work(transaction, evaluation.evaluation_id).await?; + let work = work.ok_or_else(|| ControlError::Invalid("sensor has no location work".into()))?; + if work.record.result_json.is_none() { + let expired = work.record.expires_at <= transaction.now; + return Ok(( + transaction, + expired.then_some(SensorEvaluationOutcome::Abandoned), + )); + } + match work.record.result_json.map(|result| result.outcome) { + Some(LocationWorkOutcome::SensorEvaluation { outcome }) => { + apply_result(transaction, evaluation, outcome).await + } + Some(LocationWorkOutcome::Superseded {}) => { + Ok((transaction, Some(SensorEvaluationOutcome::Abandoned))) + } + _ => Err(ControlError::Invalid( + "sensor has an invalid location result".into(), + )), + } +} + +async fn apply_result( + mut transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + outcome: CustomSensorOutcome, +) -> Result<(ControlTransaction, Option), ControlError> { + let (custom, run_keys) = match outcome { + CustomSensorOutcome::Completed { cursor, run_keys } => (cursor, run_keys), + CustomSensorOutcome::Skipped { cursor, .. } => (cursor, Vec::new()), + CustomSensorOutcome::Failed => { + return Ok(( + transaction, + Some(SensorEvaluationOutcome::Failed { + code: "custom_sensor_failed".into(), + }), + )); + } + CustomSensorOutcome::TimedOut => { + return Ok((transaction, Some(SensorEvaluationOutcome::TimedOut))); + } + }; + let mut cursor = evaluation.cursor.clone(); + cursor.custom = custom; + let mut run_ids: Vec = Vec::new(); + let run_keys: BTreeSet = run_keys.into_iter().collect(); + for key in run_keys { + let run_id; + (transaction, run_id) = queue_sensor_run(transaction, evaluation, &key).await?; + run_ids.push(run_id); + } + Ok(( + transaction, + Some(SensorEvaluationOutcome::Completed { cursor, run_ids }), + )) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs new file mode 100644 index 0000000000..c94e5577c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs @@ -0,0 +1,65 @@ +//! Attempts and cursor commits use Recall's serialized commit-order sequence. + +use serde_json::json; +use uuid::Uuid; + +use crate::sensors::models::{ + SensorCursor, SensorEvaluationFact, SensorEvaluationOutcome, SensorEvaluationRequest, +}; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +pub(crate) async fn append_evaluation( + transaction: ControlTransaction, + request: &SensorEvaluationRequest, + outcome: SensorEvaluationOutcome, +) -> Result<(ControlTransaction, (Uuid, i64)), ControlError> { + let phase = outcome.phase(); + let event_id = Uuid::new_v5(&request.evaluation_id, phase.as_bytes()); + let occurred_at = transaction.now; + let fact = SensorEvaluationFact { + request: request.clone(), + outcome, + }; + let payload = json!(fact); + let (transaction, appended) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("sensor/{}", request.trigger_id), + event_type: "sensor_evaluation", + producer: "scheduler", + occurred_at, + run_id: None, + payload: payload.clone(), + }, + ) + .await?; + if appended.conflict.is_some() { + return Err(ControlError::Invalid( + "sensor evaluation conflicts with a durable fact".into(), + )); + } + let storage_id = appended + .storage_id + .ok_or_else(|| ControlError::Invalid("sensor evaluation has no storage cursor".into()))?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_sensor_evaluation_events (event_id,storage_id,evaluation_id,trigger_id,phase,occurred_at,fact_json) \ + VALUES ($1,$2,$3,$4,$5,$6,$7)", + vec![Parameter::Id(event_id), Parameter::Integer(storage_id), Parameter::Id(request.evaluation_id), + Parameter::Id(request.trigger_id), Parameter::Text(phase.into()), Parameter::Time(occurred_at), Parameter::Json(payload)]).await?; + Ok((transaction, (event_id, storage_id))) +} + +pub(crate) async fn commit_cursor( + transaction: ControlTransaction, + (trigger_id, event_id, storage_id): (Uuid, Uuid, i64), + cursor: &SensorCursor, +) -> Result { + let (transaction, _) = query(transaction, + "INSERT INTO sqb_sensor_cursor_commits (event_id,storage_id,trigger_id,cursor_json) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(event_id), Parameter::Integer(storage_id), Parameter::Id(trigger_id), Parameter::Json(json!(cursor))]).await?; + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs new file mode 100644 index 0000000000..720f46154e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod attempts; +pub(crate) mod completion; +pub(crate) mod conditions; +pub(crate) mod custom; +pub(crate) mod facts; +pub(crate) mod nodes; +pub(crate) mod platform; +pub(crate) mod requests; +pub(crate) mod runs; diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs new file mode 100644 index 0000000000..c7f7a97b92 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -0,0 +1,95 @@ +//! Successful rebuilds require earlier immutable start evidence from the same run claim. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::runs::models::{NodeOutcome, RunChange, RunChangeRequest}; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{NodeSensorEvent, SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn consume_node_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + selection: &BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT storage_id,event_id,run_id,payload FROM recall_events \ + WHERE producer='scheduler' AND event_type='run_node_finished' \ + AND invocation_id='scheduler/' || run_id AND storage_id>$1 \ + ORDER BY storage_id LIMIT 100", + vec![Parameter::Integer(evaluation.cursor.event_storage_id)], + ) + .await?; + let mut cursor = evaluation.cursor.clone(); + let mut run_ids: Vec = Vec::new(); + for row in rows { + let event: NodeSensorEvent = decode_row(row)?; + cursor.event_storage_id = event.storage_id; + let request: RunChangeRequest = serde_json::from_value(event.payload["change"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if request.run_id != event.run_id || request.event_id != event.event_id { + return Err(ControlError::Invalid("node event identity mismatch".into())); + } + let RunChange::NodeFinished { + node_key, + outcome: NodeOutcome::Success, + .. + } = &request.change + else { + continue; + }; + if !selection.contains(node_key) { + continue; + } + let started; + (transaction, started) = + prior_start(transaction, evaluation, &request, event.storage_id).await?; + if started { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; + run_ids.push(run_id); + } + } + Ok((transaction, (cursor, run_ids))) +} + +async fn prior_start( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + request: &RunChangeRequest, + storage_id: i64, +) -> Result<(ControlTransaction, bool), ControlError> { + let (RunChange::NodeFinished { node_key, .. }, Some(claim_id)) = + (&request.change, request.claim_id) + else { + return Ok((transaction, false)); + }; + let (transaction, rows) = query( + transaction, + "SELECT s.event_id FROM sqb_runs r JOIN recall_events s ON s.run_id=CAST(r.run_id AS TEXT) \ + WHERE s.invocation_id=$1 AND s.run_id=$2 AND s.producer='scheduler' \ + AND s.event_type='run_node_started' AND s.storage_id<$3 \ + AND (s.payload -> 'change' ->> 'claim_id')=$4 \ + AND (s.payload -> 'change' -> 'change' ->> 'node_key')=$5 \ + AND r.run_id=$6 AND r.location_id=$7 AND r.target=$8 AND r.kind<>'group' LIMIT 1", + vec![ + Parameter::Text(format!("scheduler/{}", request.run_id)), + Parameter::Text(request.run_id.to_string()), + Parameter::Integer(storage_id), + Parameter::Text(claim_id.to_string()), + Parameter::Text(node_key.clone()), + Parameter::Id(request.run_id), + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + ], + ) + .await?; + Ok((transaction, !rows.is_empty())) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs new file mode 100644 index 0000000000..6ce610bf03 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -0,0 +1,36 @@ +//! One bounded platform page yields matched run requests and a storage-order cursor. + +use uuid::Uuid; + +use crate::platform::main::read_platform_events::read_platform_events; +use crate::platform::models::PlatformEventKind; +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::{SensorCursor, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(crate) async fn consume_platform_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + kinds: &std::collections::BTreeSet, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, events) = + read_platform_events(transaction, evaluation.cursor.event_storage_id).await?; + let cursor = SensorCursor { + event_storage_id: events + .last() + .map_or(evaluation.cursor.event_storage_id, |event| event.storage_id), + ..evaluation.cursor.clone() + }; + let mut run_ids: Vec = Vec::new(); + for event in events + .iter() + .filter(|event| kinds.contains(&event.payload.platform.kind())) + { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; + run_ids.push(run_id); + } + Ok((transaction, (cursor, run_ids))) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs new file mode 100644 index 0000000000..237959031b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs @@ -0,0 +1,62 @@ +//! A sensor run key gates queue publication within the cursor/outcome transaction. + +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::runs::main::frozen_job_request::frozen_job_request; +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::{JobRunRequest, RunOrigin}; +use crate::sensors::models::SensorEvaluationRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn queue_sensor_run( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + run_key: &str, +) -> Result<(ControlTransaction, Uuid), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT run_id FROM sqb_run_requests WHERE trigger_id=$1 AND run_key=$2", + vec![ + Parameter::Id(evaluation.trigger_id), + Parameter::Text(run_key.into()), + ], + ) + .await?; + if let Some(row) = rows.into_iter().next() { + return Ok(( + transaction, + serde_json::from_value(row["run_id"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + )); + } + let run_id = Uuid::new_v5(&evaluation.trigger_id, run_key.as_bytes()); + let context = AuditContext { + actor: "system:sensor".into(), + granting_role: "system:scheduler".into(), + request_id: evaluation.evaluation_id, + ip_address: None, + }; + let authored = JobRunRequest { + run_id, + location_id: evaluation.location_id, + target: evaluation.target.clone(), + job_key: evaluation.definition.job_key.clone(), + reason: "Sensor condition matched".into(), + priority: 0, + window: None, + }; + let (transaction, mut frozen) = frozen_job_request(transaction, &context, &authored).await?; + if frozen.snapshot_id != evaluation.snapshot_id { + return Err(ControlError::Fenced); + } + frozen.origin = RunOrigin::Sensor; + frozen.trigger_id = Some(evaluation.trigger_id); + let transaction = queue_run(transaction, &frozen).await?; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_run_requests (trigger_id,run_key,evaluation_id,run_id) VALUES ($1,$2,$3,$4)", + vec![Parameter::Id(evaluation.trigger_id), Parameter::Text(run_key.into()), Parameter::Id(evaluation.evaluation_id), Parameter::Id(run_id)]).await?; + Ok((transaction, run_id)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs new file mode 100644 index 0000000000..32b7e00e3c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs @@ -0,0 +1,95 @@ +//! Terminal projections carry the durable finish cursor; late observations cannot move it. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::constants::EXECUTION_NODE_BATCH_SIZE; +use crate::sensors::models::{ + RunEventKind, SensorCursor, SensorEvaluationRequest, TerminalSensorRun, +}; +use crate::store::errors::ControlError; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn consume_run_events( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, + (job_key, kinds): (&str, &BTreeSet), + executed_successfully: Option<&BTreeSet>, +) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { + let (mut transaction, rows) = query( + transaction, + "SELECT run_id,status,last_event_storage_id FROM sqb_runs \ + WHERE location_id=$1 AND target=$2 AND job_key=$3 AND last_event_storage_id>$4 \ + AND status IN ('success','warn','partial','failed','cancelled','presumed_failed') \ + ORDER BY last_event_storage_id LIMIT 100", + vec![ + Parameter::Id(evaluation.location_id), + Parameter::Text(evaluation.target.clone()), + Parameter::Text(job_key.into()), + Parameter::Integer(evaluation.cursor.event_storage_id), + ], + ) + .await?; + let runs: Vec = + rows.into_iter().map(decode_row).collect::>()?; + let mut eligible: BTreeSet = runs.iter().map(|run| run.run_id).collect(); + if let Some(nodes) = executed_successfully { + (transaction, eligible) = matching_executions(transaction, eligible, nodes).await?; + } + let mut cursor = evaluation.cursor.clone(); + let mut run_ids: Vec = Vec::new(); + for run in runs { + cursor.event_storage_id = run.last_event_storage_id; + if eligible.contains(&run.run_id) && kinds.iter().any(|kind| kind.matches(run.status)) { + let run_id; + (transaction, run_id) = + queue_sensor_run(transaction, evaluation, &run.run_id.to_string()).await?; + run_ids.push(run_id); + } + } + Ok((transaction, (cursor, run_ids))) +} + +/// Each chunk intersects the previous matches: every selected node must qualify. +async fn matching_executions( + mut transaction: ControlTransaction, + mut eligible: BTreeSet, + nodes: &BTreeSet, +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let nodes: Vec<&String> = nodes.iter().collect(); + for chunk in nodes.chunks(EXECUTION_NODE_BATCH_SIZE) { + if eligible.is_empty() { + break; + } + let mut parameters: Vec = eligible.iter().copied().map(Parameter::Id).collect(); + let run_parameters = (1..=parameters.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + let node_parameters = (parameters.len() + 1..=parameters.len() + chunk.len()) + .map(|index| format!("${index}")) + .collect::>() + .join(","); + parameters.extend(chunk.iter().map(|key| Parameter::Text((*key).to_string()))); + let rows; + (transaction, rows) = query(transaction, &format!( + "SELECT r.run_id,r.status,r.last_event_storage_id FROM sqb_runs r \ + WHERE r.run_id IN ({run_parameters}) AND \ + (SELECT COUNT(*) FROM sqb_run_nodes n WHERE n.run_id=r.run_id AND n.attempt=1 \ + AND n.node_key IN ({node_parameters}) AND n.status='success' AND n.ended_at IS NOT NULL \ + AND ((r.kind<>'group' AND n.started_at IS NOT NULL) OR \ + (r.kind='group' AND EXISTS(SELECT 1 FROM sqb_runs c \ + JOIN sqb_run_nodes cn ON cn.run_id=c.run_id AND cn.node_key=n.node_key AND cn.attempt=1 \ + WHERE c.parent_run_id=r.run_id AND c.location_id=r.location_id AND c.target=r.target \ + AND cn.status='success' AND cn.started_at IS NOT NULL AND cn.ended_at IS NOT NULL))))={}", + chunk.len()), parameters).await?; + let matches: Vec = + rows.into_iter().map(decode_row).collect::>()?; + eligible = matches.into_iter().map(|run| run.run_id).collect(); + } + Ok((transaction, eligible)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/constants.rs b/crates/sqlbuild-orchestration/src/sensors/constants.rs new file mode 100644 index 0000000000..c74670d166 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/constants.rs @@ -0,0 +1,5 @@ +//! Bound query parameters independently of the size of an authored selection. + +pub(crate) const EXECUTION_NODE_BATCH_SIZE: usize = 500; +pub(crate) const CUSTOM_SENSOR_RECHECK_SECONDS: i64 = 1; +pub const MAX_SENSOR_RUN_REQUESTS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs new file mode 100644 index 0000000000..db7c36bb49 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs @@ -0,0 +1,82 @@ +//! Commit the requested phase before evaluation, so crashes count as attempts. + +use chrono::TimeDelta; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; +use crate::sensors::_helpers::custom::request_evaluation_work; +use crate::sensors::_helpers::facts::append_evaluation; +use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn begin_sensor_evaluation( + store: &ControlStore, + lease: &SchedulerLease, + trigger_id: Uuid, +) -> Result, ControlError> { + let transaction = check_lease(begin(store, TransactionMode::Write).await?, lease).await?; + let (transaction, sensor) = read_sensor(transaction, trigger_id).await?; + let Some(sensor) = sensor else { + return Ok(None); + }; + if sensor.status != TriggerStatus::Running { + return Ok(None); + } + let now = transaction.now; + let (transaction, cursor) = read_sensor_cursor(transaction, trigger_id).await?; + let (mut transaction, previous) = last_attempt(transaction, trigger_id).await?; + if let Some((previous, terminal)) = previous { + if !terminal { + if previous.request.epoch == lease.epoch + && previous.request.snapshot_id == sensor.snapshot_id + && previous.request.cursor == cursor + { + commit(transaction).await?; + return Ok(Some(previous.request)); + } + (transaction, _) = append_evaluation( + transaction, + &previous.request, + SensorEvaluationOutcome::Abandoned, + ) + .await?; + } + let next = previous.request.requested_at + + TimeDelta::seconds(i64::from( + sensor.definition_json.minimum_interval_seconds.get(), + )); + if next > now { + commit(defer_sensor(transaction, trigger_id, next).await?).await?; + return Ok(None); + } + } + let definition_fingerprint = sha256_hex( + &serde_json::to_vec(&sensor.definition_json) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + let request = SensorEvaluationRequest { + evaluation_id: Uuid::now_v7(), + trigger_id: sensor.trigger_id, + location_id: sensor.location_id, + target: sensor.target, + snapshot_id: sensor.snapshot_id, + definition: sensor.definition_json, + definition_fingerprint, + cursor, + epoch: lease.epoch, + requested_at: now, + }; + let (transaction, _) = + append_evaluation(transaction, &request, SensorEvaluationOutcome::Requested).await?; + let transaction = request_evaluation_work(transaction, &request).await?; + commit(check_lease(transaction, lease).await?).await?; + Ok(Some(request)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs new file mode 100644 index 0000000000..5fbd35834d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -0,0 +1,28 @@ +//! Cursor, outcome, deduplicated requests and queued runs commit together. + +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::completion::{finish_evaluation, pending_evaluation}; +use crate::sensors::_helpers::conditions::consume_condition; +use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +pub async fn complete_builtin_sensor( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result { + let Some(transaction) = pending_evaluation(store, lease, evaluation).await? else { + return Ok(0); + }; + let (transaction, (cursor, run_ids)) = consume_condition(transaction, evaluation).await?; + let count = run_ids.len(); + finish_evaluation( + transaction, + lease, + evaluation, + SensorEvaluationOutcome::Completed { cursor, run_ids }, + ) + .await?; + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs new file mode 100644 index 0000000000..98d23dd900 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_custom_sensor.rs @@ -0,0 +1,42 @@ +//! A location result cannot queue runs or advance a cursor until a fenced scheduler commits it. + +use chrono::TimeDelta; + +use crate::scheduler::main::check_lease::check_lease; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::_helpers::attempts::defer_sensor; +use crate::sensors::_helpers::completion::{finish_evaluation, pending_evaluation}; +use crate::sensors::_helpers::custom::read_evaluation_result; +use crate::sensors::constants::CUSTOM_SENSOR_RECHECK_SECONDS; +use crate::sensors::models::{SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest}; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::models::ControlStore; + +pub async fn complete_custom_sensor( + store: &ControlStore, + lease: &SchedulerLease, + evaluation: &SensorEvaluationRequest, +) -> Result { + if !matches!(evaluation.definition.when, SensorCondition::Custom { .. }) { + return Err(ControlError::Invalid( + "custom sensor evaluation required".into(), + )); + } + let Some(transaction) = pending_evaluation(store, lease, evaluation).await? else { + return Ok(0); + }; + let (transaction, outcome) = read_evaluation_result(transaction, evaluation).await?; + let Some(outcome) = outcome else { + let next = transaction.now + TimeDelta::seconds(CUSTOM_SENSOR_RECHECK_SECONDS); + let transaction = defer_sensor(transaction, evaluation.trigger_id, next).await?; + commit(check_lease(transaction, lease).await?).await?; + return Ok(0); + }; + let count = match &outcome { + SensorEvaluationOutcome::Completed { run_ids, .. } => run_ids.len(), + _ => 0, + }; + finish_evaluation(transaction, lease, evaluation, outcome).await?; + Ok(count) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs new file mode 100644 index 0000000000..e819eb77e1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs @@ -0,0 +1,7 @@ +pub mod begin_sensor_evaluation; +pub mod complete_builtin_sensor; +pub mod complete_custom_sensor; +pub(crate) mod publish_sensors; +pub(crate) mod read_sensor_cursor; +pub(crate) mod reset_sensor_cursor; +pub(crate) mod validate_sensors; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs new file mode 100644 index 0000000000..3f2029c481 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/publish_sensors.rs @@ -0,0 +1,37 @@ +//! Sensor publication keeps operator overrides and immutable cursor history across deployments. + +use serde_json::json; +use uuid::Uuid; + +use crate::declarations::models::Definitions; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn publish_sensors( + mut transaction: ControlTransaction, + snapshot: &SnapshotRecord, + definitions: &Definitions, +) -> Result { + for (name, sensor) in &definitions.sensors { + let trigger_id = Uuid::new_v5( + &snapshot.location_id, + &serde_json::to_vec(&(&snapshot.target, "sensor", name)) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ); + (transaction, _) = query(transaction, + "INSERT INTO sqb_triggers (trigger_id,location_id,target,kind,name,job_key,definition_json,snapshot_id) \ + VALUES ($1,$2,$3,'sensor',$4,$5,$6,$7) ON CONFLICT (trigger_id) DO UPDATE \ + SET job_key=excluded.job_key,definition_json=excluded.definition_json,snapshot_id=excluded.snapshot_id,removed_in_snapshot_id=NULL", + vec![Parameter::Id(trigger_id), Parameter::Id(snapshot.location_id), Parameter::Text(snapshot.target.clone()), + Parameter::Text(name.clone()), Parameter::Text(sensor.job_key.clone()), Parameter::Json(json!(sensor)), Parameter::Id(snapshot.snapshot_id)]).await?; + let status = sensor.default_status.projection(); + (transaction, _) = query(transaction, + "INSERT INTO sqb_trigger_state (trigger_id,status,started_at,next_due_at,updated_at) VALUES ($1,$2,$3,$3,$3) \ + ON CONFLICT (trigger_id) DO UPDATE SET status=COALESCE(sqb_trigger_state.status_override,excluded.status), \ + next_due_at=excluded.next_due_at,updated_at=excluded.updated_at", + vec![Parameter::Id(trigger_id), Parameter::Text(status.as_str().into()), Parameter::Time(snapshot.published_at)]).await?; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs b/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs new file mode 100644 index 0000000000..ab2c060dd5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/read_sensor_cursor.rs @@ -0,0 +1,24 @@ +//! Operator reads and evaluations share the latest immutable cursor commit. + +use uuid::Uuid; + +use crate::sensors::models::SensorCursor; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_sensor_cursor( + transaction: ControlTransaction, + trigger_id: Uuid, +) -> Result<(ControlTransaction, SensorCursor), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT cursor_json FROM sqb_sensor_cursor_commits WHERE trigger_id=$1 ORDER BY storage_id DESC LIMIT 1", + vec![Parameter::Id(trigger_id)]).await?; + let cursor = rows + .first() + .map(|row| serde_json::from_value(row["cursor_json"].clone())) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))? + .unwrap_or_default(); + Ok((transaction, cursor)) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs b/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs new file mode 100644 index 0000000000..e447028c30 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/reset_sensor_cursor.rs @@ -0,0 +1,41 @@ +//! Operator reset appends a cursor fact without mutating previous commits or run keys. + +use serde_json::json; +use uuid::Uuid; + +use crate::sensors::_helpers::facts::commit_cursor; +use crate::sensors::models::SensorCursor; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::models::{ControlEvent, ControlTransaction}; + +pub(crate) async fn reset_sensor_cursor( + transaction: ControlTransaction, + (trigger_id, request_id): (Uuid, Uuid), +) -> Result { + let cursor = SensorCursor::default(); + let event_id = Uuid::new_v5(&request_id, b"sensor-cursor-reset"); + let occurred_at = transaction.now; + let (transaction, appended) = append_control_event( + transaction, + ControlEvent { + event_id, + invocation_id: format!("sensor/{trigger_id}"), + event_type: "sensor_cursor_reset", + producer: "scheduler", + occurred_at, + run_id: None, + payload: json!({"trigger_id": trigger_id, "cursor": cursor}), + }, + ) + .await?; + if appended.conflict.is_some() { + return Err(ControlError::Invalid( + "sensor cursor reset conflicts with a durable fact".into(), + )); + } + let storage_id = appended + .storage_id + .ok_or_else(|| ControlError::Invalid("sensor cursor reset has no storage cursor".into()))?; + commit_cursor(transaction, (trigger_id, event_id, storage_id), &cursor).await +} diff --git a/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs new file mode 100644 index 0000000000..da488cc75d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/validate_sensors.rs @@ -0,0 +1,53 @@ +//! Native publication validates every sensor before any snapshot pointer moves. + +use crate::declarations::models::Definitions; +use crate::sensors::models::SensorCondition; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +pub(crate) fn validate_sensors( + definitions: &Definitions, + input: &SnapshotInput, +) -> Result<(), ControlError> { + for (name, sensor) in &definitions.sensors { + let valid_condition = match &sensor.when { + SensorCondition::Custom { + function, + code_hash, + } => !function.trim().is_empty() && !code_hash.is_empty(), + SensorCondition::PlatformEvent { kinds } => !kinds.is_empty(), + SensorCondition::NodeEvent { selection, kinds } => { + !kinds.is_empty() + && !selection.is_empty() + && selection.iter().all(|key| input.nodes.contains_key(key)) + } + SensorCondition::RunEvent { + job_key, + kinds, + executed_successfully, + } => { + !kinds.is_empty() + && definitions + .jobs + .get(job_key) + .or_else(|| definitions.groups.get(job_key).map(|group| &group.job)) + .is_some_and(|source| { + executed_successfully.as_ref().is_none_or(|nodes| { + !nodes.is_empty() && nodes.is_subset(&source.selection) + }) + }) + } + }; + if name.trim().is_empty() + || sensor.description.trim().is_empty() + || !valid_condition + || (!definitions.jobs.contains_key(&sensor.job_key) + && !definitions.groups.contains_key(&sensor.job_key)) + { + return Err(ControlError::Invalid(format!( + "sensor {name} needs a description, a known job and nonempty event kinds; executed_successfully must select nonempty source job/group nodes; node_event must select nonempty snapshot nodes" + ))); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/sensors/mod.rs b/crates/sqlbuild-orchestration/src/sensors/mod.rs new file mode 100644 index 0000000000..9580a2dd16 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/mod.rs @@ -0,0 +1,10 @@ +//! Sensor attempts and cursor commits are immutable facts, not mutable lifecycle rows. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/sensors/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs new file mode 100644 index 0000000000..38fbfced25 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -0,0 +1,202 @@ +//! Published conditions and immutable attempts share one sensor abstraction. + +use std::collections::BTreeSet; +use std::num::NonZeroU32; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::declarations::models::TriggerStatus; +use crate::platform::models::PlatformEventKind; +use crate::runs::models::RunOutcome; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum SensorStatus { + #[default] + Running, + Stopped, +} + +impl SensorStatus { + #[must_use] + pub const fn projection(self) -> TriggerStatus { + match self { + Self::Running => TriggerStatus::Running, + Self::Stopped => TriggerStatus::Paused, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SensorCondition { + PlatformEvent { + kinds: BTreeSet, + }, + RunEvent { + job_key: String, + kinds: BTreeSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + executed_successfully: Option>, + }, + NodeEvent { + selection: BTreeSet, + kinds: BTreeSet, + }, + Custom { + function: String, + code_hash: String, + }, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum NodeEventKind { + Success, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum RunEventKind { + Completed, + Success, + Warn, + Partial, + Failed, + Cancelled, + PresumedFailed, +} + +impl RunEventKind { + #[must_use] + pub const fn matches(self, outcome: RunOutcome) -> bool { + matches!( + (self, outcome), + (Self::Completed, _) + | (Self::Success, RunOutcome::Success) + | (Self::Warn, RunOutcome::Warn) + | (Self::Partial, RunOutcome::Partial) + | (Self::Failed, RunOutcome::Failed) + | (Self::Cancelled, RunOutcome::Cancelled) + | (Self::PresumedFailed, RunOutcome::PresumedFailed) + ) + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct TerminalSensorRun { + pub(crate) run_id: Uuid, + pub(crate) status: RunOutcome, + pub(crate) last_event_storage_id: i64, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct NodeSensorEvent { + pub(crate) event_id: Uuid, + pub(crate) run_id: Uuid, + pub(crate) storage_id: i64, + pub(crate) payload: serde_json::Value, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct SensorDefinition { + pub description: String, + pub job_key: String, + pub when: SensorCondition, + #[schema(value_type = u32, minimum = 1)] + pub minimum_interval_seconds: NonZeroU32, + #[serde(default)] + pub default_status: SensorStatus, +} + +/// Storage order is authoritative even when a fact's occurrence time predates earlier pages. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorCursor { + pub event_storage_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub custom: Option, +} + +/// Location-owned Python returns data; only the fenced scheduler may apply it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum CustomSensorOutcome { + Completed { + cursor: Option, + run_keys: Vec, + }, + Skipped { + cursor: Option, + reason: String, + }, + Failed, + TimedOut, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorEvaluationRequest { + pub evaluation_id: Uuid, + pub trigger_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub snapshot_id: Uuid, + pub definition: SensorDefinition, + pub definition_fingerprint: String, + pub cursor: SensorCursor, + pub epoch: i64, + pub requested_at: DateTime, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case", deny_unknown_fields)] +pub enum SensorEvaluationOutcome { + Requested, + Completed { + cursor: SensorCursor, + run_ids: Vec, + }, + Failed { + code: String, + }, + TimedOut, + Abandoned, +} + +impl SensorEvaluationOutcome { + #[must_use] + pub const fn phase(&self) -> &'static str { + match self { + Self::Requested => "requested", + Self::Completed { .. } => "completed", + Self::Failed { .. } => "failed", + Self::TimedOut => "timed_out", + Self::Abandoned => "abandoned", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SensorEvaluationFact { + pub request: SensorEvaluationRequest, + pub outcome: SensorEvaluationOutcome, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct DueSensor { + pub(crate) trigger_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, + pub(crate) snapshot_id: Uuid, + pub(crate) definition_json: SensorDefinition, + pub(crate) status: TriggerStatus, +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs new file mode 100644 index 0000000000..3ce8bbd7c3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/content.rs @@ -0,0 +1,82 @@ +//! Canonical content addresses and compressed immutable bodies. + +use std::collections::BTreeMap; +use std::io::{Read, Write}; + +use flate2::{Compression, read::GzDecoder, write::GzEncoder}; +use serde_json::Value; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::snapshots::models::{NodeContent, PreparedNode, SnapshotNode}; +use crate::store::errors::ControlError; + +fn canonical(value: &Value) -> Value { + match value { + Value::Object(values) => { + let sorted: BTreeMap<_, _> = values + .iter() + .map(|(key, value)| (key.clone(), canonical(value))) + .collect(); + Value::Object(sorted.into_iter().collect()) + } + Value::Array(values) => Value::Array(values.iter().map(canonical).collect()), + _ => value.clone(), + } +} + +pub(crate) fn json_bytes(value: &Value) -> Result, ControlError> { + serde_json::to_vec(&canonical(value)).map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) fn compress(bytes: &[u8]) -> Result, ControlError> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(bytes)?; + Ok(encoder.finish()?) +} + +pub(crate) fn decompress(value: &Value) -> Result, ControlError> { + let bytes: Vec = serde_json::from_value(value.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let mut output: Vec = Vec::new(); + GzDecoder::new(bytes.as_slice()).read_to_end(&mut output)?; + Ok(output) +} + +pub(crate) fn node_content(node: &SnapshotNode) -> Result, ControlError> { + let mut sources: BTreeMap<&'static str, (String, &str)> = BTreeMap::new(); + for (kind, text) in [ + ("raw_sql", &node.raw_sql), + ("compiled_sql", &node.compiled_sql), + ("python_source", &node.python_source), + ] { + if let Some(text) = text { + sources.insert(kind, (sha256_hex(text.as_bytes()), text)); + } + } + let source_hashes: BTreeMap<_, _> = sources + .iter() + .map(|(kind, (hash, _))| (*kind, hash)) + .collect(); + let definition = json_bytes( + &serde_json::json!({"kind": node.kind, "definition": node.definition, "sources": source_hashes}), + )?; + Ok(NodeContent { + definition, + sources, + }) +} + +pub(crate) fn prepare_node(node: &SnapshotNode) -> Result { + let content = node_content(node)?; + let sources = content + .sources + .into_iter() + .map(|(kind, (hash, text))| Ok((kind, (hash, compress(text.as_bytes())?)))) + .collect::>()?; + Ok(PreparedNode { + kind: node.kind.clone(), + hash: sha256_hex(&content.definition), + definition: compress(&content.definition)?, + sources, + }) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs new file mode 100644 index 0000000000..8853fb96b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod content; +pub(crate) mod preparation; +pub(crate) mod publication; +pub(crate) mod reading; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs new file mode 100644 index 0000000000..310f723503 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs @@ -0,0 +1,74 @@ +//! Validate and content-address a successful compile before holding a database write lock. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::snapshots::_helpers::content::{compress, json_bytes, prepare_node}; +use crate::snapshots::models::{PreparedNode, PreparedSnapshot, SnapshotInput}; +use crate::store::errors::ControlError; + +pub(crate) fn prepare_snapshot(input: &SnapshotInput) -> Result { + if input.target.trim().is_empty() + || input.environment_hash.is_empty() + || input.generation <= 0 + || input + .nodes + .iter() + .any(|(key, node)| key.is_empty() || node.kind.is_empty()) + { + return Err(ControlError::Invalid( + "target, environment, positive generation, and named nodes are required".into(), + )); + } + let mut nodes: BTreeMap = BTreeMap::new(); + for (key, node) in &input.nodes { + nodes.insert(key.clone(), prepare_node(node)?); + } + let hashes: BTreeMap = nodes + .iter() + .map(|(key, node)| (key.clone(), node.hash.clone())) + .collect(); + let documents = prepare_documents(input)?; + let document_hashes: BTreeMap<_, _> = documents + .iter() + .map(|(kind, (hash, _))| (kind, hash)) + .collect(); + let manifest_digest = sha256_hex(&json_bytes(&serde_json::json!({ + "nodes": hashes, "documents": document_hashes, "environment_hash": input.environment_hash, + "image_id": input.image_id, "target": input.target, "sqb_version": input.sqb_version, + "git_revision": input.git_revision, "git_dirty": input.git_dirty, "git_author": input.git_author, + }))?); + Ok(PreparedSnapshot { + nodes, + hashes, + documents, + manifest_digest, + }) +} + +fn prepare_documents( + input: &SnapshotInput, +) -> Result)>, ControlError> { + let mut documents: BTreeMap)> = BTreeMap::new(); + for (kind, value) in &input.documents { + if ![ + "jobs", + "groups", + "triggers", + "roles", + "lineage", + "columns", + "providers", + ] + .contains(&kind.as_str()) + { + return Err(ControlError::Invalid(format!( + "unknown snapshot document {kind}" + ))); + } + let bytes = json_bytes(value)?; + documents.insert(kind.clone(), (sha256_hex(&bytes), compress(&bytes)?)); + } + Ok(documents) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs new file mode 100644 index 0000000000..e05f14da2d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/publication.rs @@ -0,0 +1,69 @@ +//! Immutable body inserts shared by snapshot publication. + +use std::collections::{BTreeMap, BTreeSet}; + +use uuid::Uuid; + +use crate::snapshots::models::{PreparedNode, SnapshotChange}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn store_nodes( + mut transaction: ControlTransaction, + snapshot_id: Uuid, + nodes: BTreeMap, +) -> Result { + for (key, node) in nodes { + (transaction, _) = query(transaction, + "INSERT INTO sqb_node_definitions (node_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(node.hash.clone()), Parameter::Bytes(node.definition)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_nodes (snapshot_id, node_key, node_kind, node_hash) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(node.kind), Parameter::Text(node.hash)]).await?; + for (kind, (hash, body)) in node.sources { + (transaction, _) = query(transaction, + "INSERT INTO sqb_source_texts (text_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_sources (snapshot_id, node_key, kind, text_hash) VALUES ($1, $2, $3, $4)", + vec![Parameter::Id(snapshot_id), Parameter::Text(key.clone()), Parameter::Text(kind.into()), Parameter::Text(hash)]).await?; + } + } + Ok(transaction) +} + +pub(crate) async fn store_changes( + mut transaction: ControlTransaction, + snapshot_id: Uuid, + previous: &BTreeMap, + hashes: &BTreeMap, +) -> Result { + let keys: BTreeSet<&String> = previous.keys().chain(hashes.keys()).collect(); + let mut sequence = 0; + for key in keys { + let before = previous.get(key); + let after = hashes.get(key); + if before == after { + continue; + } + let change = SnapshotChange { + change_kind: match (before, after) { + (None, _) => "added", + (_, None) => "removed", + _ => "changed", + } + .into(), + subject_key: key.clone(), + before_hash: before.cloned(), + after_hash: after.cloned(), + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_changes (snapshot_id, change_seq, change_kind, subject_key, before_hash, after_hash) \ + VALUES ($1,$2,$3,$4,$5,$6)", vec![Parameter::Id(snapshot_id), Parameter::Integer(sequence), + Parameter::Text(change.change_kind), Parameter::Text(change.subject_key), + Parameter::OptionalText(change.before_hash), Parameter::OptionalText(change.after_hash)]).await?; + sequence += 1; + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs new file mode 100644 index 0000000000..a350003e29 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/reading.rs @@ -0,0 +1,74 @@ +//! Decode stored bodies and attach source text without evaluating it. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; +use serde_json::Value; +use uuid::Uuid; + +use crate::snapshots::_helpers::content::decompress; +use crate::snapshots::models::{SnapshotNode, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::types::Row; + +pub(crate) async fn read_current_record( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.location_id, s.target, s.generation, s.manifest_digest, s.published_at \ + FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + Ok(( + transaction, + rows.into_iter().next().map(decode_row).transpose()?, + )) +} + +fn decode_body(value: &Value) -> Result { + serde_json::from_slice(&decompress(value)?) + .map_err(|error| ControlError::Invalid(error.to_string())) +} + +pub(crate) fn read_nodes(rows: Vec) -> Result, ControlError> { + read_bodies(rows, "node_key") +} + +fn read_bodies( + rows: Vec, + key: &str, +) -> Result, ControlError> { + rows.into_iter() + .map(|row| { + let body: T = decode_body(&row["body"])?; + Ok((row[key].as_str().unwrap_or_default().to_owned(), body)) + }) + .collect() +} + +pub(crate) fn attach_sources( + mut nodes: BTreeMap, + rows: Vec, +) -> Result, ControlError> { + for row in rows { + let key = row["node_key"].as_str().unwrap_or_default(); + let node = nodes + .get_mut(key) + .ok_or_else(|| ControlError::Invalid("snapshot source has no node".into()))?; + let text = String::from_utf8(decompress(&row["body"])?) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + match row["kind"].as_str() { + Some("raw_sql") => node.raw_sql = Some(text), + Some("compiled_sql") => node.compiled_sql = Some(text), + Some("python_source") => node.python_source = Some(text), + _ => return Err(ControlError::Invalid("unknown stored source kind".into())), + } + } + Ok(nodes) +} + +pub(crate) fn read_documents(rows: Vec) -> Result, ControlError> { + read_bodies(rows, "kind") +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs new file mode 100644 index 0000000000..81889f713b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot.rs @@ -0,0 +1,49 @@ +//! Read one consistent generation without locking out snapshot publishers. + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::{attach_sources, read_current_record}; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::snapshots::models::{CurrentSnapshot, SnapshotChange}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::decode_row::decode_row; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// The last successfully published generation, or none before the first successful compile. +pub async fn current_snapshot( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let (transaction, record) = read_current_record( + begin(store, TransactionMode::Read).await?, + (location_id, target), + ) + .await?; + let Some(record) = record else { + return Ok(None); + }; + let (transaction, nodes) = read_snapshot_nodes(transaction, record.snapshot_id, None).await?; + let (transaction, rows) = query(transaction, + "SELECT s.node_key, s.kind, t.body FROM sqb_snapshot_sources s JOIN sqb_source_texts t ON t.text_hash=s.text_hash WHERE s.snapshot_id=$1", + vec![Parameter::Id(record.snapshot_id)]).await?; + let nodes = attach_sources(nodes, rows)?; + let (transaction, documents) = + read_snapshot_documents(transaction, record.snapshot_id, None).await?; + let (transaction, rows) = query(transaction, + "SELECT change_kind, subject_key, before_hash, after_hash FROM sqb_snapshot_changes WHERE snapshot_id=$1 ORDER BY change_seq", + vec![Parameter::Id(record.snapshot_id)]).await?; + let changes: Vec = + rows.into_iter().map(decode_row).collect::>()?; + commit(transaction).await?; + Ok(Some(CurrentSnapshot { + record, + nodes, + documents, + changes, + })) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs new file mode 100644 index 0000000000..58f44ee9a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs @@ -0,0 +1,23 @@ +//! Reconcile publication identity without reading node definitions, source bodies or documents. + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_current_record; +use crate::snapshots::models::SnapshotRecord; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub async fn current_snapshot_record( + store: &ControlStore, + location_id: Uuid, + target: &str, +) -> Result, ControlError> { + let (transaction, record) = read_current_record( + begin(store, TransactionMode::Read).await?, + (location_id, target), + ) + .await?; + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs new file mode 100644 index 0000000000..6918426ec2 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -0,0 +1,8 @@ +pub mod current_snapshot; +pub mod current_snapshot_record; +pub mod next_remote_generation; +pub mod publish_remote_snapshot; +pub mod publish_snapshot; +pub(crate) mod read_snapshot_documents; +pub(crate) mod read_snapshot_nodes; +pub mod verify_execution; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs new file mode 100644 index 0000000000..3c4f974037 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/next_remote_generation.rs @@ -0,0 +1,36 @@ +//! Reconcile publication generations from the authenticated server, including lost acknowledgements. + +use uuid::Uuid; + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::snapshots::models::SnapshotRecord; + +pub async fn next_remote_generation( + server_url: &str, + bearer: &str, + location_id: Uuid, + target: &str, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&[ + "api", + "locations", + &location_id.to_string(), + "targets", + target, + "snapshots", + "metadata", + ])?; + match client_response::(connection.client.get(url)).await { + Ok(current) => current + .generation + .checked_add(1) + .ok_or(ClientError::Configuration( + "snapshot generation is exhausted", + )), + Err(ClientError::Rejected(status)) if status == reqwest::StatusCode::NOT_FOUND => Ok(1), + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs new file mode 100644 index 0000000000..681cc0f61e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_remote_snapshot.rs @@ -0,0 +1,23 @@ +//! A code location sends retained compiled data over HTTP and never opens the control store. + +use crate::api::errors::ClientError; +use crate::api::main::client_response::client_response; +use crate::api::main::connect_client::connect_client; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; + +pub async fn publish_remote_snapshot( + server_url: &str, + bearer: &str, + input: &SnapshotInput, +) -> Result { + let connection = connect_client(server_url, bearer)?; + let url = connection.endpoint(&[ + "api", + "locations", + &input.location_id.to_string(), + "targets", + &input.target, + "snapshots", + ])?; + client_response(connection.client.post(url).json(input)).await +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs new file mode 100644 index 0000000000..c34cf82ab9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/publish_snapshot.rs @@ -0,0 +1,93 @@ +//! Publish a complete generation and its current pointer in one transaction. + +use std::collections::BTreeMap; + +use serde_json::Value; +use uuid::Uuid; + +use crate::declarations::main::publish_definitions::publish_definitions; +use crate::declarations::main::snapshot_definitions::snapshot_definitions; +use crate::snapshots::_helpers::preparation::prepare_snapshot; +use crate::snapshots::_helpers::publication::{store_changes, store_nodes}; +use crate::snapshots::models::{PreparedSnapshot, SnapshotInput, SnapshotRecord}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Reject stale generations without changing the last good snapshot. +pub async fn publish_snapshot( + store: &ControlStore, + input: &SnapshotInput, +) -> Result { + let definitions = snapshot_definitions(input)?; + let PreparedSnapshot { + nodes, + hashes, + documents, + manifest_digest, + } = prepare_snapshot(input)?; + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, previous) = query(transaction, + "SELECT s.generation FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + if previous + .first() + .and_then(|row| row.get("generation")) + .and_then(Value::as_i64) + .is_some_and(|generation| generation >= input.generation) + { + return Err(ControlError::StaleGeneration); + } + let (mut transaction, previous) = query(transaction, + "SELECT n.node_key, n.node_hash FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone())]).await?; + let previous: BTreeMap = previous + .into_iter() + .map(|row| { + ( + row["node_key"].as_str().unwrap_or_default().into(), + row["node_hash"].as_str().unwrap_or_default().into(), + ) + }) + .collect(); + let record = SnapshotRecord { + snapshot_id: Uuid::now_v7(), + location_id: input.location_id, + target: input.target.clone(), + generation: input.generation, + manifest_digest, + published_at: transaction.now, + }; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshots (snapshot_id, location_id, target, generation, git_revision, git_dirty, git_author, image_id, \ + environment_hash, sqb_version, snapshot_schema_version, manifest_digest, published_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,1,$11,$12)", vec![ + Parameter::Id(record.snapshot_id), Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Integer(input.generation), Parameter::OptionalText(input.git_revision.clone()), Parameter::Boolean(input.git_dirty), + Parameter::OptionalText(input.git_author.clone()), Parameter::OptionalText(input.image_id.clone()), + Parameter::Text(input.environment_hash.clone()), Parameter::Text(input.sqb_version.clone()), + Parameter::Text(record.manifest_digest.clone()), Parameter::Time(record.published_at), + ]).await?; + transaction = store_nodes(transaction, record.snapshot_id, nodes).await?; + for (kind, (hash, body)) in documents { + (transaction, _) = query(transaction, + "INSERT INTO sqb_documents (document_hash, body) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![Parameter::Text(hash.clone()), Parameter::Bytes(body)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_snapshot_documents (snapshot_id, kind, document_hash) VALUES ($1, $2, $3)", + vec![Parameter::Id(record.snapshot_id), Parameter::Text(kind), Parameter::Text(hash)]).await?; + } + transaction = store_changes(transaction, record.snapshot_id, &previous, &hashes).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_current_snapshots (location_id, target, snapshot_id, since) VALUES ($1,$2,$3,$4) \ + ON CONFLICT (location_id, target) DO UPDATE SET snapshot_id=excluded.snapshot_id, since=excluded.since", + vec![Parameter::Id(input.location_id), Parameter::Text(input.target.clone()), + Parameter::Id(record.snapshot_id), Parameter::Time(record.published_at)]).await?; + transaction = publish_definitions(transaction, &record, definitions).await?; + commit(transaction).await?; + Ok(record) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs new file mode 100644 index 0000000000..45a03182f8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs @@ -0,0 +1,36 @@ +//! Decode immutable documents within the caller's consistent snapshot transaction. + +use std::collections::BTreeMap; + +use serde_json::Value; +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_documents; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_snapshot_documents( + transaction: ControlTransaction, + snapshot_id: Uuid, + kinds: Option<&[&str]>, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + if kinds.is_some_and(<[&str]>::is_empty) { + return Ok((transaction, BTreeMap::new())); + } + let mut parameters = vec![Parameter::Id(snapshot_id)]; + let filter = kinds.map_or(String::new(), |keys| { + parameters.extend(keys.iter().map(|key| Parameter::Text((*key).into()))); + format!( + " AND d.kind IN ({})", + (2..parameters.len() + 1) + .map(|index| format!("${index}")) + .collect::>() + .join(",") + ) + }); + let (transaction, rows) = query(transaction, + &format!("SELECT d.kind, b.body FROM sqb_snapshot_documents d JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.snapshot_id=$1{filter}"), + parameters).await?; + Ok((transaction, read_documents(rows)?)) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs new file mode 100644 index 0000000000..33240705d0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs @@ -0,0 +1,36 @@ +//! Read retained node definitions without expanding their display-only source bodies. + +use std::collections::{BTreeMap, BTreeSet}; + +use uuid::Uuid; + +use crate::snapshots::_helpers::reading::read_nodes; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +pub(crate) async fn read_snapshot_nodes( + transaction: ControlTransaction, + snapshot_id: Uuid, + selection: Option<&BTreeSet>, +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + if selection.is_some_and(BTreeSet::is_empty) { + return Ok((transaction, BTreeMap::new())); + } + let mut parameters = vec![Parameter::Id(snapshot_id)]; + let filter = selection.map_or(String::new(), |keys| { + parameters.extend(keys.iter().cloned().map(Parameter::Text)); + format!( + " AND n.node_key IN ({})", + (2..parameters.len() + 1) + .map(|index| format!("${index}")) + .collect::>() + .join(",") + ) + }); + let (transaction, rows) = query(transaction, + &format!("SELECT n.node_key, d.body FROM sqb_snapshot_nodes n JOIN sqb_node_definitions d ON d.node_hash=n.node_hash WHERE n.snapshot_id=$1{filter} ORDER BY n.node_key"), + parameters).await?; + Ok((transaction, read_nodes(rows)?)) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs new file mode 100644 index 0000000000..d4d2c00236 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs @@ -0,0 +1,50 @@ +//! Refuse execution when a worker's local compilation differs from the pinned publication. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::runs::models::RunExecution; +use crate::snapshots::_helpers::content::node_content; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +/// Display documents and publication generations are not execution inputs. +pub fn verify_execution( + input: &SnapshotInput, + expected: &RunExecution, +) -> Result<(), ControlError> { + let request = &expected.claim.request; + if input.location_id != request.location_id || input.target != request.target { + return Err(ControlError::Invalid( + "worker location or target differs from the pinned run".into(), + )); + } + if input.image_id != request.image_id { + return Err(ControlError::Invalid( + "worker image differs from the pinned run".into(), + )); + } + if input.environment_hash != expected.environment_hash { + return Err(ControlError::Invalid( + "worker environment differs from the pinned snapshot".into(), + )); + } + if input.sqb_version != expected.sqb_version { + return Err(ControlError::Invalid( + "worker SQLBuild version differs from the pinned snapshot".into(), + )); + } + if input.nodes.len() != expected.node_hashes.len() { + return Err(ControlError::Invalid( + "worker node set differs from the pinned snapshot".into(), + )); + } + for (key, node) in &input.nodes { + let hash = sha256_hex(&node_content(node)?.definition); + if expected.node_hashes.get(key) != Some(&hash) { + return Err(ControlError::Invalid( + "worker node content differs from the pinned snapshot".into(), + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/mod.rs new file mode 100644 index 0000000000..04ae56c307 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +#[path = "../../tests/snapshots/test_contracts.rs"] +mod tests; diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs new file mode 100644 index 0000000000..20d474c674 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -0,0 +1,86 @@ +//! Snapshot inputs are data only: no executable user objects cross into the control plane. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +/// A compiled node and its display-only source. SQL and Python are never run by the server. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotNode { + pub kind: String, + pub definition: Value, + pub raw_sql: Option, + pub compiled_sql: Option, + pub python_source: Option, +} + +/// A location's successful compile for one target. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotInput { + pub location_id: Uuid, + pub target: String, + pub generation: i64, + pub git_revision: Option, + pub git_dirty: bool, + pub git_author: Option, + pub image_id: Option, + pub environment_hash: String, + pub sqb_version: String, + pub nodes: BTreeMap, + pub documents: BTreeMap, +} + +/// Immutable publication metadata. Generations increase independently per location and target. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotRecord { + pub snapshot_id: Uuid, + pub location_id: Uuid, + pub target: String, + pub generation: i64, + pub manifest_digest: String, + pub published_at: DateTime, +} + +/// Node-level changes do not duplicate unchanged definitions or source text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct SnapshotChange { + pub change_kind: String, + pub subject_key: String, + pub before_hash: Option, + pub after_hash: Option, +} + +/// The current successful snapshot, read with a consistent generation and change set. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CurrentSnapshot { + pub record: SnapshotRecord, + pub nodes: BTreeMap, + pub documents: BTreeMap, + pub changes: Vec, +} + +/// Canonical identity is shared by publication and worker verification; only publication compresses. +#[derive(Debug)] +pub(crate) struct NodeContent<'a> { + pub(crate) definition: Vec, + pub(crate) sources: BTreeMap<&'static str, (String, &'a str)>, +} + +#[derive(Debug)] +pub(crate) struct PreparedNode { + pub(crate) kind: String, + pub(crate) hash: String, + pub(crate) definition: Vec, + pub(crate) sources: BTreeMap<&'static str, (String, Vec)>, +} + +#[derive(Debug)] +pub(crate) struct PreparedSnapshot { + pub(crate) nodes: BTreeMap, + pub(crate) hashes: BTreeMap, + pub(crate) documents: BTreeMap)>, + pub(crate) manifest_digest: String, +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/database.rs b/crates/sqlbuild-orchestration/src/store/_helpers/database.rs new file mode 100644 index 0000000000..433dd5b495 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/database.rs @@ -0,0 +1,154 @@ +//! Typed binding and row decoding for the two control-store engines. + +use chrono::{DateTime, SecondsFormat, Utc}; +use rusqlite::types::{Value as SqliteValue, ValueRef}; +use serde_json::Value; +use sqlbuild_recall::schema::models::ColumnType; +use sqlx_core::{column::Column, row::Row as _, type_info::TypeInfo, value::ValueRef as _}; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::Parameter; +use crate::store::types::Row; + +pub(crate) fn sqlite_value(parameter: &Parameter) -> SqliteValue { + match parameter { + Parameter::Null(_) => SqliteValue::Null, + Parameter::Text(value) => SqliteValue::Text(value.clone()), + Parameter::OptionalText(value) => { + value.clone().map_or(SqliteValue::Null, SqliteValue::Text) + } + Parameter::Bytes(value) => SqliteValue::Blob(value.clone()), + Parameter::Integer(value) => SqliteValue::Integer(*value), + Parameter::Real(value) => SqliteValue::Real(*value), + Parameter::Json(value) => SqliteValue::Text(value.to_string()), + Parameter::Boolean(value) => SqliteValue::Integer(i64::from(*value)), + Parameter::Id(value) => SqliteValue::Text(value.to_string()), + Parameter::OptionalId(value) => value.map_or(SqliteValue::Null, |value| { + SqliteValue::Text(value.to_string()) + }), + Parameter::Time(value) => { + SqliteValue::Text(value.to_rfc3339_opts(SecondsFormat::Micros, true)) + } + Parameter::OptionalTime(value) => value.map_or(SqliteValue::Null, |value| { + SqliteValue::Text(value.to_rfc3339_opts(SecondsFormat::Micros, true)) + }), + } +} + +pub(crate) fn sqlite_query( + connection: &rusqlite::Connection, + sql: &str, + parameters: &[Parameter], +) -> Result, ControlError> { + let values: Vec = parameters.iter().map(sqlite_value).collect(); + let mut statement = connection.prepare(sql)?; + let columns: Vec = statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(); + let declared_types: Vec> = statement + .columns() + .into_iter() + .map(|column| column.decl_type().map(str::to_ascii_uppercase)) + .collect(); + for (index, value) in values.into_iter().enumerate() { + let name = format!("${}", index + 1); + let position = statement + .parameter_index(&name)? + .ok_or_else(|| ControlError::Invalid(format!("missing SQL parameter {name}")))?; + statement.raw_bind_parameter(position, value)?; + } + let mut query = statement.raw_query(); + let mut result: Vec = Vec::new(); + while let Some(row) = query.next()? { + let mut mapped = Row::new(); + for (index, name) in columns.iter().enumerate() { + let value = match row.get_ref(index)? { + ValueRef::Null => Value::Null, + ValueRef::Integer(value) if declared_types[index].as_deref() == Some("BOOLEAN") => { + Value::Bool(value != 0) + } + ValueRef::Integer(value) => Value::from(value), + ValueRef::Real(value) => Value::from(value), + ValueRef::Text(value) => Value::from(String::from_utf8_lossy(value).into_owned()), + ValueRef::Blob(value) => Value::from(value.to_vec()), + }; + let value = if declared_types[index].as_deref() == Some("JSON") { + value.as_str().map_or_else( + || Ok(value.clone()), + |text| { + serde_json::from_str(text) + .map_err(|error| ControlError::Invalid(error.to_string())) + }, + )? + } else { + value + }; + mapped.insert(name.clone(), value); + } + result.push(mapped); + } + Ok(result) +} + +pub(crate) async fn postgres_query( + transaction: &mut sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>, + sql: &str, + parameters: Vec, +) -> Result, ControlError> { + let mut query = sqlx_core::query::query(sql); + for parameter in parameters { + query = match parameter { + Parameter::Null(kind) => match kind { + ColumnType::Text => query.bind(None::), + ColumnType::Timestamp => query.bind(None::>), + ColumnType::Json => query.bind(None::), + ColumnType::Integer => query.bind(None::), + ColumnType::Boolean => query.bind(None::), + ColumnType::Real => query.bind(None::), + ColumnType::Blob => query.bind(None::>), + }, + Parameter::Text(value) => query.bind(value), + Parameter::OptionalText(value) => query.bind(value), + Parameter::Bytes(value) => query.bind(value), + Parameter::Integer(value) => query.bind(value), + Parameter::Real(value) => query.bind(value), + Parameter::Json(value) => query.bind(value), + Parameter::Boolean(value) => query.bind(value), + Parameter::Id(value) => query.bind(value), + Parameter::OptionalId(value) => query.bind(value), + Parameter::Time(value) => query.bind(value), + Parameter::OptionalTime(value) => query.bind(value), + }; + } + let mut result: Vec = Vec::new(); + for row in query.fetch_all(&mut **transaction).await? { + let mut mapped = Row::new(); + for (index, column) in row.columns().iter().enumerate() { + let raw = row.try_get_raw(index)?; + let value = if raw.is_null() { + Value::Null + } else { + match column.type_info().name() { + "BOOL" => Value::from(row.try_get::(index)?), + "INT8" => Value::from(row.try_get::(index)?), + "INT4" => Value::from(row.try_get::(index)?), + "FLOAT8" => Value::from(row.try_get::(index)?), + "UUID" => Value::from(row.try_get::(index)?.to_string()), + "TIMESTAMPTZ" => Value::from( + row.try_get::, _>(index)? + .to_rfc3339_opts(SecondsFormat::Micros, true), + ), + "JSONB" | "JSON" => row.try_get::(index)?, + "BYTEA" => Value::from(row.try_get::, _>(index)?), + _ => Value::from(row.try_get::(index)?), + } + }; + mapped.insert(column.name().to_owned(), value); + } + result.push(mapped); + } + Ok(result) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs new file mode 100644 index 0000000000..6199e096da --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/migration.rs @@ -0,0 +1,51 @@ +//! One schema template, rendered only at the database type boundary. + +use semver::Version; + +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::store::types::Row; + +pub(crate) const SCHEMA: &str = include_str!("../schema.sql"); +pub(crate) const LOCATION_INSTANCES: &str = include_str!("../location_instances.sql"); +pub(crate) const LOCATION_WORK: &str = include_str!("../location_work.sql"); +pub(crate) const PLATFORM_SINK: &str = include_str!("../platform_sink.sql"); +pub(crate) const SENSORS: &str = include_str!("../sensors.sql"); +pub(crate) const RUN_SENSORS: &str = include_str!("../run_sensors.sql"); +pub(crate) const EXECUTED_NODES: &str = include_str!("../executed_nodes.sql"); +pub(crate) const NODE_SENSORS: &str = include_str!("../node_sensors.sql"); +pub(crate) const WAREHOUSE_FRESHNESS: &str = include_str!("../warehouse_freshness.sql"); + +pub(crate) fn verify_migrations(rows: &[Row], digests: &[String]) -> Result { + let current = Version::parse(env!("CARGO_PKG_VERSION")) + .map_err(|error| ControlError::Schema(error.to_string()))?; + for (index, row) in rows.iter().enumerate() { + let minimum = row + .get("min_server_version") + .and_then(|value| value.as_str()) + .ok_or_else(|| { + ControlError::Schema("migration lacks a minimum server version".into()) + })?; + let minimum = + Version::parse(minimum).map_err(|error| ControlError::Schema(error.to_string()))?; + if row.get("version").and_then(|value| value.as_u64()) != Some(index as u64 + 1) + || minimum > current + || row.get("checksum").and_then(|value| value.as_str()) + != digests.get(index).map(String::as_str) + || index >= digests.len() + { + return Err(ControlError::Schema( + "unknown migration, checksum mismatch, or minimum server version not met".into(), + )); + } + } + Ok(current) +} + +pub(crate) fn render(store: &ControlStore, sql: &str) -> String { + let postgres = matches!(store, ControlStore::Postgres(_)); + sql.replace("{uuid}", if postgres { "UUID" } else { "TEXT" }) + .replace("{time}", if postgres { "TIMESTAMPTZ" } else { "TEXT" }) + .replace("{binary}", if postgres { "BYTEA" } else { "BLOB" }) + .replace("{json}", if postgres { "JSONB" } else { "JSON" }) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs new file mode 100644 index 0000000000..08fcf1b81c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod database; +pub(crate) mod migration; +pub(crate) mod open_sqlite; +pub(crate) mod recall_rows; +pub(crate) mod recall_schema; diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs new file mode 100644 index 0000000000..9408ebaa14 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/open_sqlite.rs @@ -0,0 +1,37 @@ +//! Configure a file-backed SQLite control store without interpreting filenames as URIs. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::store::errors::ControlError; + +pub(crate) fn sqlite_path(url: &str) -> Result { + let path = Path::new(url.strip_prefix("sqlite://").unwrap_or(url)).to_path_buf(); + if path.as_os_str().is_empty() + || path == Path::new(":memory:") + || url.contains("://") && !url.starts_with("sqlite://") + { + return Err(ControlError::Invalid( + "control-store URL must name PostgreSQL or a SQLite file".into(), + )); + } + Ok(path) +} + +pub(crate) fn open_sqlite(path: PathBuf) -> Result<(), ControlError> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + let connection = rusqlite::Connection::open_with_flags( + path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE + | rusqlite::OpenFlags::SQLITE_OPEN_CREATE + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + connection.busy_timeout(Duration::from_secs(30))?; + connection.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL")?; + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs new file mode 100644 index 0000000000..e1827a832f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/recall_rows.rs @@ -0,0 +1,111 @@ +//! Insert native Recall rows and identify retry conflicts within a control transaction. + +use std::collections::HashSet; + +use chrono::{DateTime, SecondsFormat}; +use sqlbuild_recall::records::models::{RecallRow, RecallValue}; +use sqlbuild_recall::schema::main::postgres_partition_statement::postgres_partition_statement; +use sqlbuild_recall::schema::main::table_columns::table_columns; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::{ColumnType, Table}; +use sqlbuild_recall::store::main::conflict_evidence::conflict_evidence; +use sqlbuild_recall::store::main::relational_insert::relational_insert; +use sqlbuild_recall::store::main::relational_lookup::relational_lookup; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlTransaction, Parameter, RecallAppendOutcome}; + +fn parameters(values: &[(&RecallValue, ColumnType)]) -> Result, ControlError> { + values + .iter() + .map(|(value, kind)| { + Ok(match (value, kind) { + (RecallValue::Null, kind) => Parameter::Null(*kind), + (RecallValue::Integer(value), ColumnType::Boolean) => { + Parameter::Boolean(*value != 0) + } + (RecallValue::Integer(value), _) => Parameter::Integer(*value), + (RecallValue::Real(value), _) => Parameter::Real(*value), + (RecallValue::Text(value), ColumnType::Json) => Parameter::Json( + serde_json::from_str(value) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + ), + (RecallValue::Text(value), ColumnType::Timestamp) => Parameter::Time( + DateTime::parse_from_rfc3339(value) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc(), + ), + (RecallValue::Text(value), _) => Parameter::Text(value.clone()), + (RecallValue::Blob(value), _) => Parameter::Bytes(value.clone()), + }) + }) + .collect() +} + +pub(crate) fn text<'a>(row: &'a RecallRow, name: &str) -> Result<&'a str, ControlError> { + match row.value(name) { + Some(RecallValue::Text(value)) => Ok(value), + _ => Err(ControlError::Invalid(format!("Recall {name} must be text"))), + } +} + +pub(crate) async fn write_row( + mut transaction: ControlTransaction, + row: &RecallRow, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let postgres = matches!(transaction.connection, Some(Connection::Postgres(_))); + if postgres && row.table != Table::Artifacts { + let date = DateTime::parse_from_rfc3339(text(row, table_spec(row.table).expiry_column)?) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc() + .date_naive(); + let statement = postgres_partition_statement(row.table, date) + .ok_or_else(|| ControlError::Invalid("Recall partition date is out of range".into()))?; + (transaction, _) = query(transaction, &statement, vec![]).await?; + } + let ingested = RecallValue::Text(transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true)); + let columns: HashSet = table_columns(row.table) + .into_iter() + .map(|column| column.name.to_owned()) + .collect(); + let insert = relational_insert((row, &columns), &ingested, postgres); + let returning = if row.table == Table::Events { + "storage_id" + } else { + "1 AS inserted" + }; + let (transaction, rows) = query( + transaction, + &format!("{} RETURNING {returning}", insert.sql), + parameters(&insert.values)?, + ) + .await?; + if let Some(found) = rows.first() { + return Ok(( + transaction, + RecallAppendOutcome { + inserted: true, + storage_id: found.get("storage_id").and_then(serde_json::Value::as_i64), + conflict: None, + }, + )); + } + let lookup = relational_lookup(row, postgres, true); + let (transaction, rows) = query(transaction, &lookup.sql, parameters(&lookup.values)?).await?; + let found = rows.first().ok_or_else(|| { + ControlError::Invalid("Recall duplicate has no matching stored key".into()) + })?; + let existing = found + .get("content_hash") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ControlError::Invalid("stored Recall row has no content hash".into()))?; + Ok(( + transaction, + RecallAppendOutcome { + inserted: false, + storage_id: found.get("storage_id").and_then(serde_json::Value::as_i64), + conflict: conflict_evidence(row, existing), + }, + )) +} diff --git a/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs b/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs new file mode 100644 index 0000000000..d63c9b5681 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/_helpers/recall_schema.rs @@ -0,0 +1,13 @@ +//! Embed Recall in the control database so run facts and projections share one transaction. + +use sqlbuild_recall::schema::main::control_table_statements::control_table_statements; +use sqlbuild_recall::schema::main::control_view_statements::control_view_statements; + +use crate::store::models::ControlStore; + +pub(crate) fn recall_schema(store: &ControlStore) -> Vec { + let postgres = matches!(store, ControlStore::Postgres(_)); + let mut statements = control_table_statements(postgres); + statements.extend(control_view_statements(postgres)); + statements +} diff --git a/crates/sqlbuild-orchestration/src/store/errors.rs b/crates/sqlbuild-orchestration/src/store/errors.rs new file mode 100644 index 0000000000..54a8a6d29d --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/errors.rs @@ -0,0 +1,22 @@ +//! Control-store failures; callers must not publish a successful state after one. + +/// A rejected control-plane operation or failed durable write. +#[derive(Debug, thiserror::Error)] +pub enum ControlError { + #[error("control store: {0}")] + Database(#[from] sqlx_core::Error), + #[error("control store: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error("control-store worker failed: {0}")] + Worker(#[from] tokio::task::JoinError), + #[error("control-store filesystem: {0}")] + Io(#[from] std::io::Error), + #[error("invalid control-plane data: {0}")] + Invalid(String), + #[error("control-store schema is unsupported: {0}")] + Schema(String), + #[error("snapshot generation is not newer than the current generation")] + StaleGeneration, + #[error("scheduler lease expired or was fenced by another holder")] + Fenced, +} diff --git a/crates/sqlbuild-orchestration/src/store/executed_nodes.sql b/crates/sqlbuild-orchestration/src/store/executed_nodes.sql new file mode 100644 index 0000000000..4bfa512dad --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/executed_nodes.sql @@ -0,0 +1 @@ +CREATE INDEX sqb_runs_parent ON sqb_runs(parent_run_id) WHERE parent_run_id IS NOT NULL; diff --git a/crates/sqlbuild-orchestration/src/store/location_instances.sql b/crates/sqlbuild-orchestration/src/store/location_instances.sql new file mode 100644 index 0000000000..04b3e8dfd5 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/location_instances.sql @@ -0,0 +1,9 @@ +CREATE TABLE sqb_location_instances ( + instance_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + connected_at {time} NOT NULL, + last_seen_at {time} NOT NULL, + health_json {json} NOT NULL +); +CREATE INDEX sqb_location_instances_location_seen ON sqb_location_instances(location_id, last_seen_at); diff --git a/crates/sqlbuild-orchestration/src/store/location_work.sql b/crates/sqlbuild-orchestration/src/store/location_work.sql new file mode 100644 index 0000000000..2f5274b305 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/location_work.sql @@ -0,0 +1,20 @@ +CREATE TABLE sqb_location_work ( + request_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + generation BIGINT NOT NULL, + request_json {json} NOT NULL, + requested_by TEXT NOT NULL, + requested_at {time} NOT NULL, + expires_at {time} NOT NULL, + claim_id {uuid}, + session_id {uuid} REFERENCES sqb_location_instances(instance_id), + claimed_until {time} +); +CREATE INDEX sqb_location_work_pending ON sqb_location_work(location_id, target, requested_at); +CREATE TABLE sqb_location_work_results ( + request_id {uuid} PRIMARY KEY REFERENCES sqb_location_work(request_id), + result_json {json} NOT NULL, + completed_at {time} NOT NULL +); diff --git a/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs b/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs new file mode 100644 index 0000000000..3143159449 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/append_control_event.rs @@ -0,0 +1,52 @@ +//! Sequence scheduler and platform facts inside the caller's serialized transaction. + +use chrono::SecondsFormat; +use serde_json::{Map, Value, json}; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::RowEnvelope; + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter, RecallAppendOutcome}; + +pub(crate) async fn append_control_event( + transaction: ControlTransaction, + event: ControlEvent<'_>, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let timestamp = event + .occurred_at + .to_rfc3339_opts(SecondsFormat::Micros, true); + let (transaction, rows) = query(transaction, + "SELECT invocation_sequence AS last_sequence, invocation_started_at AS started_at FROM recall_events \ + WHERE invocation_id=$1 ORDER BY invocation_sequence DESC LIMIT 1", + vec![Parameter::Text(event.invocation_id.clone())]).await?; + let previous = rows.first(); + let sequence = previous + .and_then(|row| row.get("last_sequence")) + .and_then(Value::as_i64) + .map_or(Some(0), |value| value.checked_add(1)) + .ok_or_else(|| ControlError::Invalid("run event sequence is exhausted".into()))?; + let envelope = RowEnvelope { + invocation_id: event.invocation_id, + invocation_started_at: previous + .and_then(|row| row.get("started_at")) + .and_then(Value::as_str) + .unwrap_or(×tamp) + .into(), + producer_version: env!("CARGO_PKG_VERSION").into(), + }; + let object = Map::from_iter([ + ("event_id".into(), json!(event.event_id)), + ("event_type".into(), json!(event.event_type)), + ("schema_version".into(), json!(1)), + ("occurred_at".into(), json!(timestamp)), + ("producer".into(), json!(event.producer)), + ("invocation_sequence".into(), json!(sequence)), + ("run_id".into(), json!(event.run_id)), + ("payload".into(), event.payload), + ]); + let row = event_object_row(&envelope, &object) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + append_recall(transaction, &row).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/append_recall.rs b/crates/sqlbuild-orchestration/src/store/main/append_recall.rs new file mode 100644 index 0000000000..349fe86d1a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/append_recall.rs @@ -0,0 +1,36 @@ +//! Append Recall facts and retry-conflict evidence to the caller's control transaction. + +use chrono::SecondsFormat; +use sqlbuild_recall::records::main::conflict_row::conflict_row; +use sqlbuild_recall::records::models::{RecallRow, RowEnvelope}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; + +use crate::store::_helpers::recall_rows::{text, write_row}; +use crate::store::errors::ControlError; +use crate::store::models::{ControlTransaction, RecallAppendOutcome}; + +pub(crate) async fn append_recall( + transaction: ControlTransaction, + row: &RecallRow, +) -> Result<(ControlTransaction, RecallAppendOutcome), ControlError> { + let (mut transaction, outcome) = write_row(transaction, row).await?; + if let Some(evidence) = &outcome.conflict + && row.table != Table::Conflicts + { + let envelope = RowEnvelope { + invocation_id: text(row, "invocation_id")?.into(), + invocation_started_at: text(row, "invocation_started_at")?.into(), + producer_version: text(row, "producer_version")?.into(), + }; + let detected_at = transaction.now.to_rfc3339_opts(SecondsFormat::Micros, true); + let conflict = conflict_row( + &envelope, + (table_spec(row.table).name, &evidence.record_key), + (&evidence.existing_sha256, &evidence.rejected_sha256), + &detected_at, + ); + (transaction, _) = write_row(transaction, &conflict).await?; + } + Ok((transaction, outcome)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/begin.rs b/crates/sqlbuild-orchestration/src/store/main/begin.rs new file mode 100644 index 0000000000..e97c51e077 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/begin.rs @@ -0,0 +1,59 @@ +//! Serialize durable appends before allocating their storage IDs. + +use std::time::Duration; + +use chrono::Utc; +use rusqlite::OpenFlags; + +use crate::store::errors::ControlError; +use crate::store::main::refresh_clock::refresh_clock; +use crate::store::models::{Connection, ControlStore, ControlTransaction, TransactionMode}; + +pub(crate) async fn begin( + store: &ControlStore, + mode: TransactionMode, +) -> Result { + let connection = match store { + ControlStore::Sqlite(path) => { + let path = path.clone(); + Connection::Sqlite( + tokio::task::spawn_blocking(move || { + let connection = rusqlite::Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + connection.busy_timeout(Duration::from_secs(30))?; + connection.execute_batch(if mode == TransactionMode::Write { + "PRAGMA foreign_keys=ON; BEGIN IMMEDIATE" + } else { + "PRAGMA foreign_keys=ON; PRAGMA query_only=ON; BEGIN" + })?; + Ok::<_, ControlError>(connection) + }) + .await??, + ) + } + ControlStore::Postgres(pool) => { + let mut transaction = pool.begin().await?; + if mode == TransactionMode::Write { + sqlx_core::query::query("SELECT pg_advisory_xact_lock(1936810572, 1)") + .execute(&mut *transaction) + .await?; + } else { + sqlx_core::query::query( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY", + ) + .execute(&mut *transaction) + .await?; + } + Connection::Postgres(transaction) + } + }; + let transaction = ControlTransaction { + connection: Some(connection), + now: Utc::now(), + }; + refresh_clock(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/commit.rs b/crates/sqlbuild-orchestration/src/store/main/commit.rs new file mode 100644 index 0000000000..f9e3cb46e0 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/commit.rs @@ -0,0 +1,18 @@ +//! Commit facts and their projections together. + +use crate::store::errors::ControlError; +use crate::store::models::{Connection, ControlTransaction}; + +pub(crate) async fn commit(mut transaction: ControlTransaction) -> Result<(), ControlError> { + match transaction + .connection + .take() + .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))? + { + Connection::Sqlite(connection) => { + tokio::task::spawn_blocking(move || connection.execute_batch("COMMIT")).await??; + } + Connection::Postgres(transaction) => transaction.commit().await?, + } + Ok(()) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/decode_row.rs b/crates/sqlbuild-orchestration/src/store/main/decode_row.rs new file mode 100644 index 0000000000..bc02200fb1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/decode_row.rs @@ -0,0 +1,12 @@ +//! Decode driver-neutral typed records through the store's shared row contract. + +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::store::errors::ControlError; +use crate::store::types::Row; + +pub(crate) fn decode_row(row: Row) -> Result { + serde_json::from_value(Value::Object(row.into_iter().collect())) + .map_err(|error| ControlError::Invalid(error.to_string())) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs b/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs new file mode 100644 index 0000000000..ced2c62b2c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/list_recall_invocations.rs @@ -0,0 +1,62 @@ +//! Discover invocation namespaces from retained Recall facts, including output-only attempts. + +use sqlbuild_recall::schema::constants::INVOCATION_TABLES; +use sqlbuild_recall::schema::main::table_spec::table_spec; + +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// The caller authorizes exact namespace roots; prefix matching escapes LIKE patterns and includes the slash boundary. +pub async fn list_recall_invocations( + store: &ControlStore, + namespaces: &[String], + limit: u32, +) -> Result, ControlError> { + if namespaces.is_empty() || limit == 0 { + return Ok(Vec::new()); + } + let mut parameters: Vec = Vec::new(); + let predicates = namespaces + .iter() + .map(|namespace| { + let exact = parameters.len() + 1; + parameters.push(Parameter::Text(namespace.clone())); + let pattern = namespace + .replace('!', "!!") + .replace('%', "!%") + .replace('_', "!_"); + parameters.push(Parameter::Text(format!("{pattern}/%"))); + format!( + "invocation_id=${exact} OR invocation_id LIKE ${} ESCAPE '!'", + exact + 1 + ) + }) + .collect::>() + .join(" OR "); + let tables = INVOCATION_TABLES + .iter() + .map(|table| { + format!( + "SELECT invocation_id, invocation_started_at FROM {} WHERE {predicates}", + table_spec(*table).name, + ) + }) + .collect::>() + .join(" UNION ALL "); + parameters.push(Parameter::Integer(i64::from(limit.min(1000)))); + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + &format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at \ + FROM ({tables}) AS records GROUP BY invocation_id ORDER BY invocation_id LIMIT ${}", + parameters.len() + ), + parameters, + ) + .await?; + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs new file mode 100644 index 0000000000..f2237bd093 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/migrate_store.rs @@ -0,0 +1,83 @@ +//! Forward-only migration under the same lock used by all other control-plane writes. + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use uuid::Uuid; + +use crate::store::_helpers::migration::{ + EXECUTED_NODES, LOCATION_INSTANCES, LOCATION_WORK, NODE_SENSORS, PLATFORM_SINK, RUN_SENSORS, + SCHEMA, SENSORS, WAREHOUSE_FRESHNESS, render, verify_migrations, +}; +use crate::store::_helpers::recall_schema::recall_schema; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; + +/// Create the schema once, or reject drift and schemas newer than this server supports. +pub async fn migrate_store(store: &ControlStore) -> Result<(), ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + (transaction, _) = query(transaction, &render(store, + "CREATE TABLE IF NOT EXISTS sqb_schema_migrations (version BIGINT PRIMARY KEY, \ + name TEXT NOT NULL, checksum TEXT NOT NULL, min_server_version TEXT NOT NULL, applied_at {time} NOT NULL)"), vec![]).await?; + let (mut transaction, rows) = query( + transaction, + "SELECT version, checksum, min_server_version FROM sqb_schema_migrations ORDER BY version", + vec![], + ) + .await?; + let recall_statements = recall_schema(store); + let digest = sha256_hex(format!("{SCHEMA}\n{}", recall_statements.join(";\n")).as_bytes()); + let additions = [ + ("location_instances", LOCATION_INSTANCES), + ("location_work", LOCATION_WORK), + ("platform_sink", PLATFORM_SINK), + ("sensors", SENSORS), + ("run_sensors", RUN_SENSORS), + ("executed_nodes", EXECUTED_NODES), + ("node_sensors", NODE_SENSORS), + ("warehouse_freshness", WAREHOUSE_FRESHNESS), + ]; + let mut digests = vec![digest.clone()]; + digests.extend(additions.iter().map(|(_, sql)| sha256_hex(sql.as_bytes()))); + let current = verify_migrations(&rows, &digests)?; + if rows.is_empty() { + for statement in SCHEMA + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + { + (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; + } + for statement in recall_statements { + (transaction, _) = query(transaction, &statement, vec![]).await?; + } + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_installation (singleton, installation_id, created_at) VALUES (1, $1, $2)", + vec![Parameter::Id(Uuid::now_v7()), Parameter::Time(now)]).await?; + (transaction, _) = query(transaction, + "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ + VALUES (1, 'control_store', $1, $2, $3)", + vec![Parameter::Text(digest), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; + } + for (index, (name, sql)) in additions.into_iter().enumerate() { + if index + 1 < rows.len() { + continue; + } + for statement in sql + .split(';') + .map(str::trim) + .filter(|statement| !statement.is_empty()) + { + (transaction, _) = query(transaction, &render(store, statement), vec![]).await?; + } + let now = transaction.now; + (transaction, _) = query(transaction, + "INSERT INTO sqb_schema_migrations (version, name, checksum, min_server_version, applied_at) \ + VALUES ($1, $2, $3, $4, $5)", + vec![Parameter::Integer(index as i64 + 2), Parameter::Text(name.into()), + Parameter::Text(digests[index + 1].clone()), Parameter::Text(current.to_string()), Parameter::Time(now)]).await?; + } + commit(transaction).await +} diff --git a/crates/sqlbuild-orchestration/src/store/main/mod.rs b/crates/sqlbuild-orchestration/src/store/main/mod.rs new file mode 100644 index 0000000000..d8119ad010 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/mod.rs @@ -0,0 +1,11 @@ +pub(crate) mod append_control_event; +pub(crate) mod append_recall; +pub(crate) mod begin; +pub(crate) mod commit; +pub(crate) mod decode_row; +pub mod list_recall_invocations; +pub mod migrate_store; +pub mod open_store; +pub(crate) mod query; +pub mod read_recall_view; +pub(crate) mod refresh_clock; diff --git a/crates/sqlbuild-orchestration/src/store/main/open_store.rs b/crates/sqlbuild-orchestration/src/store/main/open_store.rs new file mode 100644 index 0000000000..db8a450f4f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/open_store.rs @@ -0,0 +1,23 @@ +//! Open a control database; no Python or warehouse connection is involved. + +use std::time::Duration; + +use crate::store::_helpers::open_sqlite::{open_sqlite, sqlite_path}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; + +/// Open PostgreSQL by URL, or SQLite by an explicit filesystem path. +pub async fn open_store(url: &str) -> Result { + if url.starts_with("postgres://") || url.starts_with("postgresql://") { + let pool = sqlx_postgres::PgPoolOptions::new() + .max_connections(10) + .acquire_timeout(Duration::from_secs(30)) + .connect(url) + .await?; + return Ok(ControlStore::Postgres(pool)); + } + let path = sqlite_path(url)?; + let configure_path = path.clone(); + tokio::task::spawn_blocking(move || open_sqlite(configure_path)).await??; + Ok(ControlStore::Sqlite(path)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/query.rs b/crates/sqlbuild-orchestration/src/store/main/query.rs new file mode 100644 index 0000000000..9e79cdc7a1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/query.rs @@ -0,0 +1,35 @@ +//! Execute inside an owned transaction, without blocking an async runtime on SQLite. + +use crate::store::_helpers::database::{postgres_query, sqlite_query}; +use crate::store::errors::ControlError; +use crate::store::models::{Connection, ControlTransaction, Parameter}; +use crate::store::types::Row; + +pub(crate) async fn query( + mut transaction: ControlTransaction, + sql: &str, + parameters: Vec, +) -> Result<(ControlTransaction, Vec), ControlError> { + let connection = transaction + .connection + .take() + .ok_or_else(|| ControlError::Invalid("transaction has already finished".into()))?; + let rows: Vec = match connection { + Connection::Sqlite(connection) => { + let sql = sql.to_owned(); + let (connection, result) = tokio::task::spawn_blocking(move || { + let result = sqlite_query(&connection, &sql, ¶meters); + (connection, result) + }) + .await?; + transaction.connection = Some(Connection::Sqlite(connection)); + result? + } + Connection::Postgres(mut connection) => { + let result = postgres_query(&mut connection, sql, parameters).await; + transaction.connection = Some(Connection::Postgres(connection)); + result? + } + }; + Ok((transaction, rows)) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs new file mode 100644 index 0000000000..a07da01efb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/read_recall_view.rs @@ -0,0 +1,44 @@ +//! Only the supported Recall views can be read; no caller-provided SQL is evaluated. + +use sqlbuild_recall::schema::constants::VIEW_NAMES; + +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::types::Row; + +/// Read authorized evidence, decoding only display text while preserving byte-exact captured content. +pub async fn read_recall_view( + store: &ControlStore, + (view, invocation_id): (&str, &str), + limit: u32, +) -> Result, ControlError> { + if !VIEW_NAMES.contains(&view) { + return Err(ControlError::Invalid("unknown Recall view".into())); + } + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, mut rows) = query( + transaction, + &format!("SELECT * FROM \"{view}\" WHERE invocation_id=$1 LIMIT $2"), + vec![ + Parameter::Text(invocation_id.into()), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + if matches!(view, "output_tail" | "artifacts") { + for row in &mut rows { + if let Some(text) = row.get_mut("text") + && text.is_array() + { + let bytes: Vec = serde_json::from_value(text.clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + *text = serde_json::Value::String(String::from_utf8_lossy(&bytes).into_owned()); + } + } + } + commit(transaction).await?; + Ok(rows) +} diff --git a/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs new file mode 100644 index 0000000000..0453de1ae3 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/main/refresh_clock.rs @@ -0,0 +1,26 @@ +//! Refresh the authoritative database clock after taking transaction locks. + +use chrono::DateTime; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{Connection, ControlTransaction}; + +pub(crate) async fn refresh_clock( + transaction: ControlTransaction, +) -> Result { + let sql = match transaction.connection { + Some(Connection::Postgres(_)) => "SELECT clock_timestamp() AS now", + _ => "SELECT strftime('%Y-%m-%dT%H:%M:%f000Z','now') AS now", + }; + let (mut transaction, rows) = query(transaction, sql, vec![]).await?; + let now = rows + .first() + .and_then(|row| row.get("now")) + .and_then(|value| value.as_str()) + .ok_or_else(|| ControlError::Invalid("database clock did not return a timestamp".into()))?; + transaction.now = DateTime::parse_from_rfc3339(now) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .to_utc(); + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/store/mod.rs b/crates/sqlbuild-orchestration/src/store/mod.rs new file mode 100644 index 0000000000..1228df7ae6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; +pub(crate) mod types; + +#[cfg(test)] +#[path = "../../tests/store/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/store/models.rs b/crates/sqlbuild-orchestration/src/store/models.rs new file mode 100644 index 0000000000..414bca79cd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/models.rs @@ -0,0 +1,71 @@ +//! Native database handles and bound values shared by control-plane operations. + +use std::path::PathBuf; + +use chrono::{DateTime, Utc}; +use sqlbuild_recall::schema::models::ColumnType; +use sqlbuild_recall::store::models::ConflictEvidence; +use uuid::Uuid; + +/// An installation's control store. Workers and locations never receive this handle. +#[derive(Clone, Debug)] +pub enum ControlStore { + Sqlite(PathBuf), + Postgres(sqlx_postgres::PgPool), +} + +/// The first copy wins; duplicate events retain their original storage cursor. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecallAppendOutcome { + pub inserted: bool, + pub storage_id: Option, + pub conflict: Option, +} + +/// A control-plane fact shares Recall's durable sequencing and conflict handling. +#[derive(Debug)] +pub(crate) struct ControlEvent<'a> { + pub(crate) event_id: Uuid, + pub(crate) invocation_id: String, + pub(crate) event_type: &'a str, + pub(crate) producer: &'static str, + pub(crate) occurred_at: DateTime, + pub(crate) run_id: Option, + pub(crate) payload: serde_json::Value, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TransactionMode { + Read, + Write, +} + +/// An owned transaction; abandoning it rolls back, including when an async task is cancelled. +#[derive(Debug)] +pub(crate) struct ControlTransaction { + pub(crate) connection: Option, + pub(crate) now: DateTime, +} + +#[derive(Debug)] +pub(crate) enum Connection { + Sqlite(rusqlite::Connection), + Postgres(sqlx_core::transaction::Transaction<'static, sqlx_postgres::Postgres>), +} + +/// Parameters keep their PostgreSQL types; SQLite uses their corresponding storage classes. +#[derive(Clone, Debug)] +pub(crate) enum Parameter { + Null(ColumnType), + Text(String), + OptionalText(Option), + Bytes(Vec), + Integer(i64), + Real(f64), + Json(serde_json::Value), + Boolean(bool), + Id(Uuid), + OptionalId(Option), + Time(DateTime), + OptionalTime(Option>), +} diff --git a/crates/sqlbuild-orchestration/src/store/node_sensors.sql b/crates/sqlbuild-orchestration/src/store/node_sensors.sql new file mode 100644 index 0000000000..372ccf6b4c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/node_sensors.sql @@ -0,0 +1,6 @@ +CREATE INDEX sqb_node_finished_cursor ON recall_events(storage_id) +WHERE producer='scheduler' AND event_type='run_node_finished'; +CREATE INDEX sqb_node_started_identity ON recall_events( + invocation_id, (payload -> 'change' ->> 'claim_id'), + (payload -> 'change' -> 'change' ->> 'node_key'), storage_id +) WHERE producer='scheduler' AND event_type='run_node_started'; diff --git a/crates/sqlbuild-orchestration/src/store/platform_sink.sql b/crates/sqlbuild-orchestration/src/store/platform_sink.sql new file mode 100644 index 0000000000..ece8bb3b6f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/platform_sink.sql @@ -0,0 +1,7 @@ +CREATE TABLE sqb_platform_sink_deliveries ( + sink_id TEXT NOT NULL, + storage_id BIGINT NOT NULL, + event_id TEXT NOT NULL, + delivered_at {time} NOT NULL, + PRIMARY KEY (sink_id, storage_id) +); diff --git a/crates/sqlbuild-orchestration/src/store/run_sensors.sql b/crates/sqlbuild-orchestration/src/store/run_sensors.sql new file mode 100644 index 0000000000..de46a16b7b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/run_sensors.sql @@ -0,0 +1,2 @@ +CREATE INDEX sqb_runs_terminal_cursor ON sqb_runs(location_id, target, job_key, last_event_storage_id) +WHERE status IN ('success', 'warn', 'partial', 'failed', 'cancelled', 'presumed_failed'); diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql new file mode 100644 index 0000000000..c4c689ca2e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -0,0 +1,264 @@ +CREATE TABLE sqb_installation ( + singleton BIGINT PRIMARY KEY CHECK (singleton = 1), + installation_id {uuid} NOT NULL, + created_at {time} NOT NULL +); +CREATE TABLE sqb_locations ( + location_id {uuid} PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + created_by TEXT NOT NULL, + created_at {time} NOT NULL, + archived_at {time}, + broken_deploy_policy TEXT NOT NULL DEFAULT 'fallback' + CHECK (broken_deploy_policy IN ('fallback', 'queue')) +); +CREATE TABLE sqb_source_texts ( + text_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_node_definitions ( + node_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_documents ( + document_hash TEXT PRIMARY KEY, + body {binary} NOT NULL +); +CREATE TABLE sqb_snapshots ( + snapshot_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + generation BIGINT NOT NULL CHECK (generation > 0), + git_revision TEXT, + git_dirty BOOLEAN NOT NULL, + git_author TEXT, + image_id TEXT, + environment_hash TEXT NOT NULL, + sqb_version TEXT NOT NULL, + snapshot_schema_version BIGINT NOT NULL, + manifest_digest TEXT NOT NULL, + published_at {time} NOT NULL, + UNIQUE (location_id, target, generation), + UNIQUE (location_id, target, snapshot_id) +); +CREATE INDEX sqb_snapshots_location_time ON sqb_snapshots(location_id, target, published_at); +CREATE TABLE sqb_snapshot_nodes ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + node_key TEXT NOT NULL, + node_kind TEXT NOT NULL, + node_hash TEXT NOT NULL REFERENCES sqb_node_definitions(node_hash), + PRIMARY KEY (snapshot_id, node_key) +); +CREATE TABLE sqb_snapshot_sources ( + snapshot_id {uuid} NOT NULL, + node_key TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('raw_sql', 'compiled_sql', 'python_source')), + text_hash TEXT NOT NULL REFERENCES sqb_source_texts(text_hash), + PRIMARY KEY (snapshot_id, node_key, kind), + FOREIGN KEY (snapshot_id, node_key) REFERENCES sqb_snapshot_nodes(snapshot_id, node_key) +); +CREATE TABLE sqb_snapshot_documents ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('jobs', 'groups', 'triggers', 'roles', 'lineage', 'columns', 'providers')), + document_hash TEXT NOT NULL REFERENCES sqb_documents(document_hash), + PRIMARY KEY (snapshot_id, kind) +); +CREATE TABLE sqb_current_snapshots ( + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL, + since {time} NOT NULL, + PRIMARY KEY (location_id, target), + FOREIGN KEY (location_id, target, snapshot_id) + REFERENCES sqb_snapshots(location_id, target, snapshot_id) +); +CREATE TABLE sqb_snapshot_changes ( + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + change_seq BIGINT NOT NULL, + change_kind TEXT NOT NULL CHECK (change_kind IN ('added', 'changed', 'removed')), + subject_key TEXT NOT NULL, + before_hash TEXT, + after_hash TEXT, + PRIMARY KEY (snapshot_id, change_seq) +); +CREATE TABLE sqb_scheduler_lease ( + name TEXT PRIMARY KEY CHECK (name = 'scheduler'), + holder {uuid} NOT NULL, + epoch BIGINT NOT NULL CHECK (epoch > 0), + expires_at {time} NOT NULL +); +CREATE TABLE sqb_jobs ( + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + job_key TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + kind TEXT NOT NULL CHECK (kind IN ('job', 'group')), + definition_json {json} NOT NULL, + overlap_policy TEXT NOT NULL CHECK (overlap_policy IN ('coalesce', 'queue', 'allow')), + PRIMARY KEY (location_id, target, job_key) +); +CREATE TABLE sqb_job_nodes ( + location_id {uuid} NOT NULL, + target TEXT NOT NULL, + job_key TEXT NOT NULL, + node_key TEXT NOT NULL, + PRIMARY KEY (location_id, target, job_key, node_key), + FOREIGN KEY (location_id, target, job_key) REFERENCES sqb_jobs(location_id, target, job_key) +); +CREATE TABLE sqb_triggers ( + trigger_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('schedule', 'sensor')), + name TEXT NOT NULL, + job_key TEXT NOT NULL, + definition_json {json} NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + removed_in_snapshot_id {uuid} REFERENCES sqb_snapshots(snapshot_id), + UNIQUE (location_id, target, kind, name) +); +CREATE TABLE sqb_trigger_state ( + trigger_id {uuid} PRIMARY KEY REFERENCES sqb_triggers(trigger_id), + status TEXT NOT NULL CHECK (status IN ('running', 'paused')), + status_override TEXT CHECK (status_override IN ('running', 'paused')), + started_at {time} NOT NULL, + cursor_json {json}, + last_tick_for {time}, + next_due_at {time}, + consecutive_failures BIGINT NOT NULL DEFAULT 0, + updated_at {time} NOT NULL +); +CREATE INDEX sqb_trigger_state_due ON sqb_trigger_state(next_due_at) WHERE status = 'running'; +CREATE TABLE sqb_trigger_state_events ( + event_id {uuid} PRIMARY KEY, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + occurred_at {time} NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + request_json {json} NOT NULL +); +CREATE INDEX sqb_trigger_state_events_time ON sqb_trigger_state_events(trigger_id, occurred_at, event_id); +CREATE TABLE sqb_ticks ( + tick_id {uuid} PRIMARY KEY, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + scheduled_for {time} NOT NULL, + evaluated_at {time} NOT NULL, + epoch BIGINT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'skipped', 'covered', 'failed', 'deferred')), + run_id {uuid}, + covered_by_run_id {uuid}, + reason TEXT, + details_json {json} NOT NULL, + UNIQUE (trigger_id, scheduled_for) +); +CREATE INDEX sqb_ticks_trigger_time ON sqb_ticks(trigger_id, scheduled_for); +CREATE TABLE sqb_runs ( + run_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + snapshot_id {uuid} NOT NULL REFERENCES sqb_snapshots(snapshot_id), + image_id TEXT, + kind TEXT NOT NULL CHECK (kind IN ('job', 'group', 'adhoc')), + origin TEXT NOT NULL CHECK (origin IN ('manual', 'schedule', 'sensor', 'rerun', 'backfill')), + job_key TEXT, + parent_run_id {uuid} REFERENCES sqb_runs(run_id), + root_run_id {uuid} NOT NULL, + trigger_id {uuid} REFERENCES sqb_triggers(trigger_id), + tick_id {uuid} REFERENCES sqb_ticks(tick_id), + rerun_of_run_id {uuid} REFERENCES sqb_runs(run_id), + requested_by TEXT NOT NULL, + priority BIGINT NOT NULL DEFAULT 0, + selection_json {json} NOT NULL, + request_json {json} NOT NULL, + write_set_digest TEXT NOT NULL, + rewrite_history BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL CHECK (status IN ('queued', 'launched', 'running', 'unresponsive', 'success', 'warn', 'partial', 'failed', 'cancelled', 'presumed_failed')), + status_reason TEXT, + queued_at {time} NOT NULL, + launched_at {time}, + started_at {time}, + ended_at {time}, + cancel_requested_at {time}, + node_counts_json {json} NOT NULL, + last_event_storage_id BIGINT NOT NULL +); +CREATE INDEX sqb_runs_location_time ON sqb_runs(location_id, target, queued_at); +CREATE INDEX sqb_runs_queue ON sqb_runs(priority DESC, queued_at) WHERE status = 'queued'; +CREATE INDEX sqb_runs_active ON sqb_runs(location_id, target, job_key) WHERE status IN ('queued', 'launched', 'running', 'unresponsive'); +CREATE TABLE sqb_run_nodes ( + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + node_key TEXT NOT NULL, + attempt BIGINT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'success', 'warn', 'failed', 'skipped', 'cancelled')), + writes BOOLEAN NOT NULL, + started_at {time}, + ended_at {time}, + rows_affected BIGINT, + error_code TEXT, + skipped_because_of TEXT, + PRIMARY KEY (run_id, node_key, attempt) +); +CREATE INDEX sqb_run_nodes_node ON sqb_run_nodes(node_key, run_id); +CREATE TABLE sqb_write_claims ( + target TEXT NOT NULL, + node_key TEXT NOT NULL, + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + claimed_at {time} NOT NULL, + PRIMARY KEY (target, node_key) +); +CREATE INDEX sqb_write_claims_run ON sqb_write_claims(run_id); +CREATE TABLE sqb_concurrency_slots ( + group_key TEXT NOT NULL, + slot BIGINT NOT NULL CHECK (slot > 0), + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + claimed_at {time} NOT NULL, + PRIMARY KEY (group_key, slot), + UNIQUE (group_key, run_id) +); +CREATE INDEX sqb_concurrency_slots_run ON sqb_concurrency_slots(run_id); +CREATE TABLE sqb_run_liveness ( + run_id {uuid} PRIMARY KEY REFERENCES sqb_runs(run_id), + worker_instance {uuid} NOT NULL, + container_id TEXT, + claimed_at {time} NOT NULL, + last_heartbeat_at {time} NOT NULL +); +CREATE TABLE sqb_principals ( + principal_id {uuid} PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + kind TEXT NOT NULL CHECK (kind IN ('person', 'service_account')), + created_at {time} NOT NULL, + disabled_at {time} +); +CREATE TABLE sqb_access_tokens ( + token_id {uuid} PRIMARY KEY, + secret_hash TEXT NOT NULL, + principal_id {uuid} REFERENCES sqb_principals(principal_id), + location_id {uuid} REFERENCES sqb_locations(location_id), + run_id {uuid} REFERENCES sqb_runs(run_id), + claim_id {uuid}, + scope_json {json} NOT NULL, + created_at {time} NOT NULL, + expires_at {time} NOT NULL, + revoked_at {time}, + CHECK ( + (principal_id IS NOT NULL AND location_id IS NULL AND run_id IS NULL AND claim_id IS NULL) + OR (principal_id IS NULL AND location_id IS NOT NULL AND run_id IS NULL AND claim_id IS NULL) + OR (principal_id IS NULL AND location_id IS NULL AND run_id IS NOT NULL AND claim_id IS NOT NULL) + ) +); +CREATE INDEX sqb_access_tokens_principal ON sqb_access_tokens(principal_id); +CREATE INDEX sqb_access_tokens_location ON sqb_access_tokens(location_id); +CREATE INDEX sqb_access_tokens_run ON sqb_access_tokens(run_id); +CREATE TABLE sqb_security_audit ( + event_id {uuid} PRIMARY KEY, + occurred_at {time} NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + granting_role TEXT NOT NULL, + request_id {uuid} NOT NULL, + ip_address TEXT, + scope_json {json} NOT NULL, + before_json {json} NOT NULL, + after_json {json} NOT NULL +); +CREATE INDEX sqb_security_audit_time ON sqb_security_audit(occurred_at, event_id); diff --git a/crates/sqlbuild-orchestration/src/store/sensors.sql b/crates/sqlbuild-orchestration/src/store/sensors.sql new file mode 100644 index 0000000000..9e1ab95213 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/sensors.sql @@ -0,0 +1,25 @@ +CREATE TABLE sqb_sensor_evaluation_events ( + event_id {uuid} PRIMARY KEY, + storage_id BIGINT NOT NULL UNIQUE, + evaluation_id {uuid} NOT NULL, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + phase TEXT NOT NULL CHECK (phase IN ('requested', 'completed', 'failed', 'timed_out', 'abandoned')), + occurred_at {time} NOT NULL, + fact_json {json} NOT NULL, + UNIQUE (evaluation_id, phase) +); +CREATE INDEX sqb_sensor_evaluation_history ON sqb_sensor_evaluation_events(trigger_id, storage_id); +CREATE TABLE sqb_sensor_cursor_commits ( + event_id {uuid} PRIMARY KEY, + storage_id BIGINT NOT NULL UNIQUE, + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + cursor_json {json} NOT NULL +); +CREATE INDEX sqb_sensor_cursor_history ON sqb_sensor_cursor_commits(trigger_id, storage_id); +CREATE TABLE sqb_run_requests ( + trigger_id {uuid} NOT NULL REFERENCES sqb_triggers(trigger_id), + run_key TEXT NOT NULL, + evaluation_id {uuid} NOT NULL, + run_id {uuid} NOT NULL REFERENCES sqb_runs(run_id), + PRIMARY KEY (trigger_id, run_key) +); diff --git a/crates/sqlbuild-orchestration/src/store/types.rs b/crates/sqlbuild-orchestration/src/store/types.rs new file mode 100644 index 0000000000..c1df9d2d41 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/types.rs @@ -0,0 +1,3 @@ +//! Internal result rows use native JSON scalars, with UUIDs and timestamps as strings. + +pub(crate) type Row = std::collections::BTreeMap; diff --git a/crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql b/crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql new file mode 100644 index 0000000000..5cdd1500e6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store/warehouse_freshness.sql @@ -0,0 +1,10 @@ +CREATE TABLE sqb_warehouse_freshness ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + source_event_id TEXT NOT NULL, + observed_at {time} NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL +); +CREATE INDEX sqb_warehouse_freshness_scope ON sqb_warehouse_freshness (location_id, target, observed_at, event_id); diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs new file mode 100644 index 0000000000..2615174d34 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -0,0 +1,3 @@ +//! Limits bound each authenticated warehouse observation transaction. + +pub const MAX_HISTORY_RECORDS: usize = 100; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs new file mode 100644 index 0000000000..a734b0440b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest_source_freshness.rs @@ -0,0 +1,103 @@ +//! Persist source identities; only legacy rows use a fixed observation-key fallback. + +use serde_json::json; +use sqlbuild_state::freshness::main::event_id::event_id as legacy_event_id; +use sqlbuild_state::freshness::models::FreshnessEventIdentity; +use uuid::Uuid; + +use crate::platform::main::require_instance::require_instance; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +pub async fn ingest_source_freshness( + store: &ControlStore, + (location_id, target): (Uuid, &str), + batch: &SourceFreshnessBatch, +) -> Result { + let invalid_identity = batch + .records + .iter() + .flat_map(|record| { + [ + &record.state_schema, + &record.source_name, + &record.run_id, + &record.strategy, + &record.value_kind, + &record.data_version_hash, + ] + }) + .any(|value| value.trim().is_empty()); + let invalid_source_id = batch.records.iter().any(|record| { + record + .event_id + .as_ref() + .is_some_and(|value| value.trim().is_empty()) + }); + if batch.generation < 1 + || batch.records.len() > MAX_HISTORY_RECORDS + || invalid_identity + || invalid_source_id + { + return Err(ControlError::Invalid(format!( + "source freshness sync needs a positive generation, at most {MAX_HISTORY_RECORDS} records and nonempty state identities", + ))); + } + let transaction = require_instance( + begin(store, TransactionMode::Write).await?, + (location_id, batch.session_id, target), + ) + .await?; + let (mut transaction, rows) = query( + transaction, + "SELECT s.snapshot_id FROM sqb_current_snapshots c JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2 AND s.generation=$3", + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(batch.generation)], + ).await?; + if rows.is_empty() { + return Err(ControlError::Fenced); + } + let mut applied = false; + for record in &batch.records { + let source_event_id = record.event_id.clone().unwrap_or_else(|| { + legacy_event_id(&FreshnessEventIdentity { + source_name: &record.source_name, + target_database: record.target_database.as_deref(), + target_schema: record.target_schema.as_deref(), + target_name: record.target_name.as_deref(), + run_id: &record.run_id, + strategy: &record.strategy, + value_kind: &record.value_kind, + data_version_hash: &record.data_version_hash, + observed_at: &record + .observed_at + .format("%Y-%m-%dT%H:%M:%S%.6f+00:00") + .to_string(), + }) + }); + let identity = serde_json::to_vec(&( + target, + &record.state_database, + &record.state_schema, + "_sqlbuild_source_freshness", + &source_event_id, + )) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let event_id = Uuid::new_v5(&location_id, &identity); + let received_at = transaction.now; + let inserted; + (transaction, inserted) = query(transaction, + "INSERT INTO sqb_warehouse_freshness (event_id,location_id,target,source_event_id,observed_at,record_json,received_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT (event_id) DO NOTHING RETURNING event_id", + vec![Parameter::Id(event_id), Parameter::Id(location_id), Parameter::Text(target.into()), + Parameter::Text(source_event_id), Parameter::Time(record.observed_at), Parameter::Json(json!(record)), Parameter::Time(received_at)]).await?; + applied |= !inserted.is_empty(); + } + commit(transaction).await?; + Ok(applied) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs new file mode 100644 index 0000000000..0ff12b7aff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -0,0 +1 @@ +pub mod ingest_source_freshness; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs new file mode 100644 index 0000000000..69fb999ac1 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/mod.rs @@ -0,0 +1,5 @@ +//! Warehouse-owned observations mirrored by authenticated code locations, never execution state. + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs new file mode 100644 index 0000000000..3a59cbc391 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -0,0 +1,102 @@ +//! Existing warehouse state rows, with their physical state-table provenance. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Preserve observations, not an inferred latest version, age status or run outcome. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseFreshnessRecord { + pub event_id: Option, + pub state_database: Option, + pub state_schema: String, + pub source_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub strategy: String, + pub value_kind: String, + pub data_version: Option, + pub data_version_hash: String, + pub observed_at: DateTime, +} + +/// A bounded page from one registered reader of a published location/target generation. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct SourceFreshnessBatch { + pub session_id: Uuid, + pub generation: i64, + pub records: Vec, +} + +/// Physical namespace of SQLBuild's source-freshness state table, never a source query. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseStateLocation { + pub database: Option, + pub schema: String, +} + +/// Private interpreter input; cursor values are data and receive adapter literal rendering. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceFreshnessRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 11]>, + /// Inclusive time bound, held fixed across pages; independent of the row-key cursor. + pub observed_since: Option>, +} + +/// Fingerprints remain history here; latest-row selection belongs to the existing planner reader. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseFingerprintRecord { + pub event_id: Option, + pub state_database: Option, + pub state_schema: String, + pub node_type: String, + pub node_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: Option, + pub run_id: String, + pub definition_hash: String, + pub version_hash: String, + pub schema_fingerprint: String, + pub definition: String, + pub metadata_json: String, + pub ts: DateTime, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FingerprintRead { + pub location: WarehouseStateLocation, + pub after: Option<[Option; 13]>, + pub observed_since: Option>, +} + +/// The private interpreter accepts only these two known state tables, never arbitrary SQL. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WarehouseHistoryRead { + SourceFreshness(SourceFreshnessRead), + Fingerprints(FingerprintRead), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum WarehouseHistoryRecord { + SourceFreshness(WarehouseFreshnessRecord), + Fingerprint(WarehouseFingerprintRecord), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WarehouseHistoryPage { + pub records: Vec, + pub next_after: Option>>, +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs new file mode 100644 index 0000000000..813c99c3ac --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/client.rs @@ -0,0 +1,73 @@ +//! A worker's authenticated HTTP requests carry only its own run and claim. + +use sqlbuild_recall::records::models::OutputChunk; +use uuid::Uuid; + +use crate::api::main::client_response::client_response; +use crate::api::models::{HeartbeatResponse, MutationResponse}; +use crate::runs::models::{OutputPublication, RunChange, RunChangeRequest, RunExecution}; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +impl WorkerClient { + /// Fetch the launcher's immutable request, not the location's latest deployment. + pub async fn execution(&self) -> Result { + let execution: RunExecution = client_response( + self.connection + .client + .get(format!("{}execution", self.run_url)), + ) + .await?; + if execution.claim.request.run_id != self.run_id + || execution.claim.claim_id != self.claim_id + { + return Err(WorkerError::Configuration( + "execution response does not match the worker's run and claim", + )); + } + Ok(execution) + } + + /// Call with the same event ID and payload when retrying an uncertain response. + pub async fn report(&self, event_id: Uuid, change: RunChange) -> Result { + let request = RunChangeRequest { + event_id, + run_id: self.run_id, + claim_id: Some(self.claim_id), + change, + }; + let result: MutationResponse = client_response( + self.connection + .client + .post(format!("{}events", self.run_url)) + .json(&request), + ) + .await?; + Ok(result.applied) + } + + /// Retry the same chunks after an uncertain response; differing retries never overwrite bytes. + pub async fn publish_output( + &self, + chunks: &[OutputChunk], + ) -> Result { + Ok(client_response( + self.connection + .client + .post(format!("{}output", self.run_url)) + .json(chunks), + ) + .await?) + } + + /// Run independently from warehouse execution; cancellation is a cooperative request. + pub async fn heartbeat(&self) -> Result { + let result: HeartbeatResponse = client_response( + self.connection + .client + .post(format!("{}heartbeat", self.run_url)), + ) + .await?; + Ok(result.cancel_requested) + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs new file mode 100644 index 0000000000..25beaf886c --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/heartbeat.rs @@ -0,0 +1,89 @@ +//! A dedicated native runtime keeps heartbeat IO independent of warehouse work and Python's GIL. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use crate::runs::models::RunExecution; +use crate::worker::errors::WorkerError; +use crate::worker::models::{WorkerClient, WorkerSession}; + +async fn heartbeat_loop( + client: WorkerClient, + cancelled: Arc, + mut stop: tokio::sync::watch::Receiver, +) { + let mut interval = tokio::time::interval(Duration::from_secs(10)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + _ = stop.changed() => return, + _ = interval.tick() => {} + } + let result = tokio::select! { + biased; + _ = stop.changed() => return, + result = client.heartbeat() => result, + }; + match result { + Ok(true) => cancelled.store(true, Ordering::Release), + Err(WorkerError::Rejected(status)) + if matches!(status.as_u16(), 401 | 403 | 404 | 410) => + { + cancelled.store(true, Ordering::Release); + return; + } + Ok(false) | Err(_) => {} + } + } +} + +pub(crate) async fn start_session( + client: WorkerClient, + execution: RunExecution, +) -> Result { + let cancelled = Arc::new(AtomicBool::new(false)); + let signal = Arc::clone(&cancelled); + let (stop, stopping) = tokio::sync::watch::channel(false); + let (ready, started) = tokio::sync::oneshot::channel(); + let thread = std::thread::Builder::new() + .name("sqb-worker-heartbeat".into()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready.send(Err(WorkerError::from(error))); + return; + } + }; + runtime.block_on(async move { + let heartbeat = match client.connection.independent() { + Ok(connection) => WorkerClient { + connection, + ..client + }, + Err(error) => { + let _ = ready.send(Err(WorkerError::from(error))); + return; + } + }; + let _ = ready.send(Ok(())); + heartbeat_loop(heartbeat, signal, stopping).await; + }); + runtime.shutdown_timeout(Duration::from_secs(1)); + })?; + let session = WorkerSession { + execution, + cancel_requested: cancelled, + stop, + thread: Some(thread), + }; + started + .await + .map_err(|_| WorkerError::Configuration("heartbeat thread stopped during startup"))??; + Ok(session) +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs new file mode 100644 index 0000000000..e3fde3c92a --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod client; +pub(crate) mod heartbeat; +pub(crate) mod recall; +pub(crate) mod session; diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs new file mode 100644 index 0000000000..b259195962 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/recall.rs @@ -0,0 +1,215 @@ +//! Reuse the native delivery thread and accounting over claim-scoped HTTP, without Python. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use sqlbuild_recall::records::models::{RecallRow, RowEnvelope}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; +use sqlbuild_recall::store::models::InsertOutcome; +use sqlbuild_recall::store::types::{RecallStore, RecallStoreOpener}; +use sqlbuild_recall::writer::constants::{BATCH_ROWS, BATCH_WINDOW_MS}; +use sqlbuild_recall::writer::models::SessionStore; + +use crate::api::constants::BODY_BYTES; +use crate::api::main::client_response::client_response; +use crate::runs::constants::RECALL_BATCH_ROWS; +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::worker::models::WorkerClient; + +#[derive(Debug)] +struct WorkerRecallOpener(WorkerClient); + +struct WorkerRecallStore { + runtime: tokio::runtime::Runtime, + client: WorkerClient, + invocation_id: String, +} + +fn failure(message: impl Into, retryable: bool) -> StoreError { + StoreError { + kind: if retryable { + StoreErrorKind::Busy + } else { + StoreErrorKind::Other + }, + message: message.into(), + } +} + +fn transport_record(row: &RecallRow) -> RecallRecord { + RecallRecord { + table: row.table, + values: table_spec(row.table) + .columns + .iter() + .filter_map(|column| { + let value = row.value(column.name)?; + Some((column.name.to_owned(), value.clone())) + }) + .collect(), + } +} + +impl WorkerClient { + /// Pass this to the native Recall session; all delivery finishes before terminal reporting. + #[must_use] + pub fn recall_store(&self) -> SessionStore { + SessionStore::Hosted(Arc::new(WorkerRecallOpener(self.clone()))) + } + + /// The server owns common envelope columns and recomputes each row's content hash. + pub async fn publish_recall( + &self, + batch: &RecallBatch, + ) -> Result { + Ok(client_response( + self.connection + .client + .post(format!("{}recall", self.run_url)) + .json(batch), + ) + .await?) + } +} + +impl RecallStoreOpener for WorkerRecallOpener { + fn open( + &self, + envelope: &RowEnvelope, + give_up: Instant, + ) -> Result, StoreError> { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| failure("Recall delivery runtime could not start", false))?; + let client = { + let _entered = runtime.enter(); + WorkerClient { + connection: self + .0 + .connection + .independent() + .map_err(|_| failure("Recall HTTP client could not start", false))?, + ..self.0.clone() + } + }; + Ok(Box::new(WorkerRecallStore { + runtime, + client, + invocation_id: envelope.invocation_id.clone(), + })) + } + + fn batch_rows(&self) -> usize { + BATCH_ROWS + } + + fn batch_window(&self) -> Duration { + Duration::from_millis(BATCH_WINDOW_MS) + } +} + +/// Native materialization emits content/reference pairs, whose proof stays in one HTTP request. +impl RecallStore for WorkerRecallStore { + fn insert(&self, rows: &[RecallRow], give_up: Instant) -> Result { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let mut batch = RecallBatch { + invocation_id: self.invocation_id.clone(), + records: Vec::new(), + }; + let base_bytes = serde_json::to_vec(&batch) + .map_err(|_| failure("Recall batch could not be encoded", false))? + .len(); + self.runtime.block_on(async { + let mut result = InsertOutcome::default(); + let mut bytes = base_bytes; + let mut offset = 0; + while offset < rows.len() { + if Instant::now() >= give_up { + return Err(failure("Recall delivery deadline expired", true)); + } + let paired = rows[offset].table == Table::Artifacts + && rows.get(offset + 1).is_some_and(|next| { + next.table == Table::ArtifactRefs + && next.value("digest") == rows[offset].value("digest") + }); + let end = offset + if paired { 2 } else { 1 }; + let records = rows[offset..end] + .iter() + .map(transport_record) + .collect::>(); + let added = serde_json::to_vec(&records) + .map_err(|_| failure("Recall records could not be encoded", false))? + .len() + - 2; + if base_bytes + added > BODY_BYTES { + return Err(failure("Recall record exceeds the HTTP body limit", false)); + } + if !batch.records.is_empty() + && (batch.records.len() + records.len() > RECALL_BATCH_ROWS + || bytes + added + 1 > BODY_BYTES) + { + (batch, result) = self.flush(batch, result, give_up).await?; + bytes = base_bytes; + } + bytes += added + usize::from(!batch.records.is_empty()); + batch.records.extend(records); + offset = end; + } + if !batch.records.is_empty() || rows.is_empty() { + (_, result) = self.flush(batch, result, give_up).await?; + } + result.delivered.sort_by_key(|(table, _)| *table); + Ok(result) + }) + } +} + +impl WorkerRecallStore { + async fn flush( + &self, + mut batch: RecallBatch, + mut result: InsertOutcome, + give_up: Instant, + ) -> Result<(RecallBatch, InsertOutcome), StoreError> { + let outcome = tokio::time::timeout_at( + tokio::time::Instant::from_std(give_up), + self.client.publish_recall(&batch), + ) + .await + .map_err(|_| failure("Recall delivery deadline expired", true))? + .map_err(|error| { + let retryable = match &error { + crate::worker::errors::WorkerError::Transport(_) => true, + crate::worker::errors::WorkerError::Rejected(status) => { + status.is_server_error() || *status == reqwest::StatusCode::TOO_MANY_REQUESTS + } + _ => false, + }; + failure(error.to_string(), retryable) + })?; + for (table, count) in outcome.delivered { + if let Some((_, previous)) = result + .delivered + .iter_mut() + .find(|(found, _)| *found == table) + { + *previous += count; + } else { + result.delivered.push((table, count)); + } + } + result.conflicts.extend(outcome.conflicts); + result.failed.extend(outcome.failed); + result.failure = result.failure.take().or(outcome.failure); + batch.records.clear(); + Ok((batch, result)) + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs b/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs new file mode 100644 index 0000000000..e7bdd3ad08 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/_helpers/session.rs @@ -0,0 +1,22 @@ +//! The execution owner drains its heartbeat before worker runtime teardown. + +use std::sync::atomic::Ordering; + +use crate::worker::models::WorkerSession; + +impl WorkerSession { + /// Cancellation and credential revocation are monotonic for one pinned worker invocation. + #[must_use] + pub fn cancel_requested(&self) -> bool { + self.cancel_requested.load(Ordering::Acquire) + } +} + +impl Drop for WorkerSession { + fn drop(&mut self) { + let _ = self.stop.send(true); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/errors.rs b/crates/sqlbuild-orchestration/src/worker/errors.rs new file mode 100644 index 0000000000..49240df632 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/errors.rs @@ -0,0 +1,27 @@ +//! Transport failures are distinct from rejected pinned execution inputs. + +#[derive(Debug, thiserror::Error)] +pub enum WorkerError { + #[error("invalid worker connection: {0}")] + Configuration(&'static str), + #[error("worker API request failed: {0}")] + Transport(#[from] reqwest::Error), + #[error("worker API rejected the request with HTTP {0}")] + Rejected(reqwest::StatusCode), + #[error("worker heartbeat could not start: {0}")] + Heartbeat(#[from] std::io::Error), + #[error(transparent)] + Verification(#[from] crate::store::errors::ControlError), + #[error("run cancellation was requested before execution")] + Cancelled, +} + +impl From for WorkerError { + fn from(error: crate::api::errors::ClientError) -> Self { + match error { + crate::api::errors::ClientError::Configuration(message) => Self::Configuration(message), + crate::api::errors::ClientError::Transport(error) => Self::Transport(error), + crate::api::errors::ClientError::Rejected(status) => Self::Rejected(status), + } + } +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs new file mode 100644 index 0000000000..15a7a4d081 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/connect_worker.rs @@ -0,0 +1,23 @@ +//! Bound worker requests have deadlines and must never follow credential-bearing redirects. + +use uuid::Uuid; + +use crate::api::main::connect_client::connect_client; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; + +/// Connect to an API origin or path prefix using the launcher's run-scoped bearer. +pub fn connect_worker( + server_url: &str, + bearer: &str, + (run_id, claim_id): (Uuid, Uuid), +) -> Result { + let connection = connect_client(server_url, bearer)?; + let run_url = connection.endpoint(&["api", "runs", &run_id.to_string(), ""])?; + Ok(WorkerClient { + connection, + run_url, + run_id, + claim_id, + }) +} diff --git a/crates/sqlbuild-orchestration/src/worker/main/mod.rs b/crates/sqlbuild-orchestration/src/worker/main/mod.rs new file mode 100644 index 0000000000..05f5586133 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/mod.rs @@ -0,0 +1,2 @@ +pub mod connect_worker; +pub mod start_worker; diff --git a/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs new file mode 100644 index 0000000000..72569448be --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/main/start_worker.rs @@ -0,0 +1,26 @@ +//! Verify the baseline publication before any run overrides or warehouse mutation. + +use uuid::Uuid; + +use crate::runs::models::RunChange; +use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::models::SnapshotInput; +use crate::worker::_helpers::heartbeat::start_session; +use crate::worker::errors::WorkerError; +use crate::worker::models::{WorkerClient, WorkerSession}; + +/// Verify the baseline compile before planning, cloning, applying run settings or executing. +pub async fn start_worker( + client: &WorkerClient, + baseline: &SnapshotInput, +) -> Result { + let expected = client.execution().await?; + verify_execution(baseline, &expected)?; + if client.heartbeat().await? { + return Err(WorkerError::Cancelled); + } + let event_id = Uuid::new_v5(&expected.claim.claim_id, b"worker-started"); + let session = start_session(client.clone(), expected).await?; + client.report(event_id, RunChange::Started).await?; + Ok(session) +} diff --git a/crates/sqlbuild-orchestration/src/worker/mod.rs b/crates/sqlbuild-orchestration/src/worker/mod.rs new file mode 100644 index 0000000000..59f273cbec --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod _helpers; +pub mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-orchestration/src/worker/models.rs b/crates/sqlbuild-orchestration/src/worker/models.rs new file mode 100644 index 0000000000..81799ca75f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/worker/models.rs @@ -0,0 +1,26 @@ +//! A worker owns only an API credential, never a control-store connection. + +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use uuid::Uuid; + +use crate::api::models::ControlClient; +use crate::runs::models::RunExecution; + +#[derive(Clone, Debug)] +pub struct WorkerClient { + pub(crate) connection: ControlClient, + pub(crate) run_url: reqwest::Url, + pub(crate) run_id: Uuid, + pub(crate) claim_id: Uuid, +} + +/// Keep this owner alive through execution and terminal reporting, then stop and join its heartbeat. +#[derive(Debug)] +pub struct WorkerSession { + pub execution: RunExecution, + pub(crate) cancel_requested: Arc, + pub(crate) stop: tokio::sync::watch::Sender, + pub(crate) thread: Option>, +} diff --git a/crates/sqlbuild-orchestration/tests/api/helpers.rs b/crates/sqlbuild-orchestration/tests/api/helpers.rs new file mode 100644 index 0000000000..9743c15603 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/helpers.rs @@ -0,0 +1,69 @@ +use std::time::Duration; + +use crate::api::main::serve_api::serve_api; +use crate::api::tests::test_types::TestApi; +use crate::store::models::ControlStore; + +pub(crate) async fn start_api(store: &ControlStore) -> TestApi { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("isolated API socket"); + let url = format!("http://{}", listener.local_addr().expect("API address")); + let (shutdown, signal) = tokio::sync::oneshot::channel(); + let store = store.clone(); + let task = tokio::spawn(serve_api(store, listener, async { + let _ = signal.await; + })); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .expect("HTTP client"); + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..250 { + assert!(!task.is_finished(), "API failed during startup"); + tokio::time::sleep(Duration::from_millis(20)).await; + let Ok(response) = client + .get(format!("{url}/api/openapi.json")) + .send() + .await + .and_then(reqwest::Response::error_for_status) + else { + continue; + }; + drop(response); + return; + } + panic!("API exhausted readiness attempts"); + }) + .await + .expect("API becomes ready"); + TestApi { + url, + shutdown, + task, + } +} + +pub(crate) async fn stop_api(api: TestApi) { + api.shutdown.send(()).expect("request API shutdown"); + tokio::time::timeout(Duration::from_secs(2), api.task) + .await + .expect("prompt API shutdown") + .expect("join API server") + .expect("API server stopped cleanly"); +} + +pub(crate) async fn read_json( + client: &reqwest::Client, + (url, bearer): (&str, &str), +) -> serde_json::Value { + let response = client + .get(url) + .bearer_auth(bearer) + .send() + .await + .expect("HTTP read"); + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + response.json().await.expect("typed JSON response") +} diff --git a/crates/sqlbuild-orchestration/tests/api/output/helpers.rs b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs new file mode 100644 index 0000000000..6d9f455567 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/output/helpers.rs @@ -0,0 +1,178 @@ +//! Worker output contract helpers inspect durable bytes on both databases. + +use crate::runs::models::OutputPublication; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::ControlStore; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; +use serde_json::json; +use sqlbuild_recall::records::models::{OmittedOutput, OutputChunk, OutputStream}; +use uuid::Uuid; + +pub(crate) fn chunk(sequence: u64, stream: OutputStream, data: &[u8]) -> OutputChunk { + OutputChunk { + sequence, + stream, + occurred_at: "2026-01-02T03:04:05Z".into(), + data: data.to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + } +} + +pub(crate) async fn stored_bytes(store: &ControlStore, invocation: &str) -> Vec<(i64, Vec)> { + let sql = "SELECT sequence, data FROM recall_output WHERE invocation_id=$1 ORDER BY sequence"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!("SQLite store") + }; + let connection = rusqlite::Connection::open(path).expect("inspect SQLite bytes"); + let mut statement = connection.prepare(sql).expect("output query"); + return statement + .query_map([invocation], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("stored rows") + .collect::>() + .expect("decode bytes"); + }; + sqlx_core::query_as::query_as(sql) + .bind(invocation) + .fetch_all(pool) + .await + .expect("inspect PostgreSQL bytes") +} + +pub(crate) async fn output_contract(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + let invocation = format!("worker/{claim_id}"); + let first = chunk(0, OutputStream::Stdout, b"orders\0\xff\n"); + let warning = chunk(1, OutputStream::Stderr, b"warning\xc3\x28"); + let omission = OutputChunk { + omitted: Some(OmittedOutput { + bytes: 43, + sequences: 2, + }), + byte_offset: first.data.len() as u64, + dropped_before: 1, + ..chunk(3, OutputStream::Stdout, b"") + }; + let chunks = [first.clone(), warning.clone(), omission]; + assert_eq!( + worker + .publish_output(&chunks) + .await + .expect("publish exact bytes"), + OutputPublication { + inserted: 3, + duplicates: 0, + conflicts: 0 + } + ); + assert_eq!( + worker + .publish_output(&chunks) + .await + .expect("retry accepted output"), + OutputPublication { + inserted: 0, + duplicates: 3, + conflicts: 0 + } + ); + let mut equivalent = first.clone(); + equivalent.occurred_at = "2026-01-02T04:04:05+01:00".into(); + assert_eq!( + worker + .publish_output(&[equivalent]) + .await + .expect("equivalent timestamp retry") + .duplicates, + 1 + ); + let mut conflict = first.clone(); + conflict.data = b"different".to_vec(); + assert_eq!( + worker + .publish_output(&[conflict]) + .await + .expect("record retry conflict") + .conflicts, + 1 + ); + assert_eq!( + stored_bytes(store, &invocation).await, + vec![ + (0, first.data.clone()), + (1, warning.data.clone()), + (3, vec![]) + ] + ); + let conflicts = read_recall_view(store, ("conflicts", &invocation), 10) + .await + .expect("durable conflicts"); + assert_eq!(conflicts.len(), 1); + let counts = read_recall_view(store, ("output_invocations", &invocation), 10) + .await + .expect("output accounting"); + assert_eq!(counts[0]["stdout_bytes"], json!(first.data.len())); + assert_eq!(counts[0]["stderr_bytes"], json!(warning.data.len())); + assert_eq!(counts[0]["truncated_bytes"], json!(43)); + + let valid = chunk(4, OutputStream::Stdout, b"must roll back"); + let mut invalid = chunk(5, OutputStream::Stderr, b"invalid timestamp"); + invalid.occurred_at = "not-a-time".into(); + for rejected in [ + vec![valid.clone(), invalid], + vec![OutputChunk { + sequence: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + sequence: i64::MAX as u64, + ..valid.clone() + }], + vec![OutputChunk { + byte_offset: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + dropped_before: u64::MAX, + ..valid.clone() + }], + vec![OutputChunk { + omitted: Some(OmittedOutput { + bytes: 1, + sequences: 0, + }), + ..valid.clone() + }], + vec![valid.clone(); 257], + vec![chunk(4, OutputStream::Stdout, &vec![0; 1_048_577])], + ] { + assert!(matches!( + worker.publish_output(&rejected).await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + } + assert_eq!( + stored_bytes(store, &invocation).await.len(), + 3, + "invalid batches are atomic" + ); + assert_eq!( + worker + .publish_output(&vec![first; 256]) + .await + .expect("chunk count boundary") + .duplicates, + 256 + ); + let boundary = chunk(4, OutputStream::Stdout, &vec![0; 1_048_576]); + assert_eq!( + worker + .publish_output(&[boundary]) + .await + .expect("byte count boundary") + .inserted, + 1 + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs new file mode 100644 index 0000000000..d6bf078a67 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/recall/helpers.rs @@ -0,0 +1,411 @@ +//! Exercise native writer delivery and authenticated retries against both physical stores. + +use std::time::Duration; + +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::store::main::read_recall_view::read_recall_view; +use crate::store::models::ControlStore; +use crate::worker::errors::WorkerError; +use crate::worker::models::WorkerClient; +use serde_json::json; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{ + ArtifactInput, InvocationContext, OutputStream, RecallRow, RecallValue, RowEnvelope, +}; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; +use sqlbuild_recall::writer::models::SessionConfig; +use uuid::Uuid; + +use crate::api::tests::output_helpers::stored_bytes; + +fn record(row: RecallRow) -> RecallRecord { + RecallRecord { + table: row.table, + values: table_spec(row.table) + .columns + .iter() + .filter_map(|column| { + row.value(column.name) + .map(|value| (column.name.to_owned(), value.clone())) + }) + .collect(), + } +} + +pub(crate) async fn recall_contract( + store: &ControlStore, + worker: &WorkerClient, + claim_id: Uuid, +) -> RecallBatch { + let envelope = RowEnvelope { + invocation_id: "execution".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "untrusted-version".into(), + }; + let invocation = format!("worker/{claim_id}/execution"); + let event = json!({ + "event_id": "same-event-id", "occurred_at": "2026-01-02T03:04:05Z", + "event_type": "statement_completed", "schema_version": 1, "producer": "executor", + "statement_id": "shared-producer-id", + "payload": {"duration_ms": 12.75, "row_count": 9, "outcome": "success"} + }); + let mut batch = RecallBatch { + invocation_id: envelope.invocation_id.clone(), + records: vec![record( + event_object_row(&envelope, event.as_object().expect("event object")) + .expect("encode event"), + )], + }; + for _ in 0..2 { + let outcome = worker + .publish_recall(&batch) + .await + .expect("retry event batch"); + assert_eq!(outcome.delivered, [(Table::Events, 1)]); + assert!(outcome.conflicts.is_empty()); + } + let events = read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read stored events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["duration_ms"], json!(12.75)); + assert_eq!(events[0]["row_count"], json!(9)); + assert_ne!(events[0]["event_id"], json!("same-event-id")); + assert_ne!(events[0]["producer_version"], json!("untrusted-version")); + batch.records[0] + .values + .insert("duration_ms".into(), RecallValue::Real(13.25)); + assert_eq!( + worker + .publish_recall(&batch) + .await + .expect("publish conflicting event") + .conflicts + .len(), + 1 + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read original event")[0]["duration_ms"], + json!(12.75) + ); + assert_eq!( + read_recall_view(store, ("conflicts", &invocation), 10) + .await + .expect("read conflict evidence") + .len(), + 1 + ); + + let mut separate = batch.clone(); + separate.invocation_id = "another-invocation".into(); + assert_eq!( + worker + .publish_recall(&separate) + .await + .expect("publish another invocation") + .delivered, + [(Table::Events, 1)] + ); + assert_eq!( + read_recall_view(store, ("statements", &invocation), 10) + .await + .expect("read first invocation statement")[0]["duration_ms"], + json!(12.75) + ); + assert_eq!( + read_recall_view( + store, + ( + "statements", + &format!("worker/{claim_id}/another-invocation") + ), + 10 + ) + .await + .expect("read second invocation statement")[0]["duration_ms"], + json!(13.25) + ); + let artifact = ArtifactInput { + digest: format!("sha256:{}", sha256_hex(b"orders\0\xff")), + kind: "evidence".into(), + media_type: "application/octet-stream".into(), + content: b"orders\0\xff".to_vec(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: "first".into(), + role: "evidence".into(), + statement_id: None, + resource_id: None, + }; + let [content, reference] = artifact_rows(&envelope, &artifact).map(record); + let artifacts = RecallBatch { + invocation_id: "execution".into(), + records: vec![content.clone(), reference.clone()], + }; + assert_eq!( + worker + .publish_recall(&artifacts) + .await + .expect("publish artifact with proof") + .delivered + .len(), + 2 + ); + let mut later_ref = reference.clone(); + later_ref + .values + .insert("ref_id".into(), RecallValue::Text("second".into())); + assert_eq!( + worker + .publish_recall(&RecallBatch { + invocation_id: "execution".into(), + records: vec![later_ref] + }) + .await + .expect("publish later artifact reference") + .delivered, + [(Table::ArtifactRefs, 1)] + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read both artifact references") + .len(), + 2 + ); + + let mut forged = content; + forged + .values + .insert("content".into(), RecallValue::Blob(b"different".to_vec())); + let mut event_next = batch.records[0].clone(); + event_next.values.insert( + "event_id".into(), + RecallValue::Text("must-roll-back".into()), + ); + let mut forged_envelope = event_next.clone(); + forged_envelope.values.insert( + "invocation_id".into(), + RecallValue::Text("scheduler/other".into()), + ); + let mut wrong_type = event_next.clone(); + wrong_type + .values + .insert("row_count".into(), RecallValue::Text("nine".into())); + let mut missing = event_next.clone(); + missing.values.remove("event_id"); + let mut bad_json = event_next.clone(); + bad_json + .values + .insert("payload".into(), RecallValue::Text("not-json".into())); + let mut bad_timestamp = event_next.clone(); + bad_timestamp + .values + .insert("occurred_at".into(), RecallValue::Text("not-a-time".into())); + for records in [ + vec![event_next.clone(), forged], + vec![event_next.clone(), forged_envelope], + vec![event_next.clone(), wrong_type], + vec![event_next.clone(), missing], + vec![event_next.clone(), bad_json], + vec![event_next.clone(), bad_timestamp], + vec![ + event_next.clone(), + RecallRecord { + table: Table::Meta, + values: Default::default(), + }, + ], + vec![ + event_next.clone(), + RecallRecord { + table: Table::Conflicts, + values: Default::default(), + }, + ], + vec![event_next; 501], + ] { + assert!(matches!( + worker + .publish_recall(&RecallBatch { + invocation_id: "execution".into(), + records + }) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + } + // A later authorization failure rolls back the preceding valid event, not just validation errors. + assert!(matches!( + worker + .publish_recall(&RecallBatch { + invocation_id: "foreign".into(), + records: vec![batch.records[0].clone(), reference] + }) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::BAD_REQUEST)) + )); + assert!( + read_recall_view(store, ("events", &format!("worker/{claim_id}/foreign")), 10) + .await + .expect("read rolled-back invocation") + .is_empty() + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read unchanged invocation") + .len(), + 1 + ); + + native_session(store, worker, claim_id).await; + large_batches(store, worker, claim_id).await; + batch +} + +async fn native_session(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + let hosted = worker.recall_store(); + let summary = tokio::task::spawn_blocking(move || { + let session = start_recall_session(SessionConfig { + store: hosted, + envelope: RowEnvelope { invocation_id: "native-writer".into(), invocation_started_at: "2026-01-02T03:04:05Z".into(), producer_version: "test-version".into() }, + context: InvocationContext { command: Some("build".into()), git_dirty: Some(true), ..Default::default() }, + retention_days: 30, shutdown_timeout: Duration::from_secs(10), diagnostics_min_severity: "debug".into(), + max_output_bytes: None, heartbeat_interval: Duration::from_secs(60), setup_budget: Duration::from_secs(2), + maintenance: false, defer_context: false, maintenance_owner: "test-worker".into(), audit_evidence: Default::default(), + }); + session.append_output(OutputStream::Stdout, b"customers\0\xff\n"); + session.append_output(OutputStream::Stderr, b"warning\xc3\x28\n"); + session.publish_sql("select 7 as order_id", Some("statement-1"), Some("orders")); + session.publish_diagnostic(&json!({"occurred_at": "2026-01-02T03:04:05Z", "severity": "warning", "logger": "sqlbuild", "source": "test", "message": "orders warning", "fields": {"count": 3}}).to_string()).expect("queue diagnostic"); + session.publish_event(&json!({"event_id": "native-event", "occurred_at": "2026-01-02T03:04:05Z", "event_type": "statement_completed", "producer": "executor", "schema_version": 1, "payload": {"duration_ms": 2.5}}).to_string()).expect("queue event"); + session.close() + }).await.expect("native writer thread"); + assert!(summary.flush_complete, "{summary:?}"); + assert_eq!(summary.total().accepted, summary.total().delivered); + let invocation = format!("worker/{claim_id}/native-writer"); + let output = stored_bytes(store, &invocation).await; + assert_eq!(output.len(), 2); + assert!( + output + .iter() + .any(|(_, bytes)| bytes == b"customers\0\xff\n") + ); + assert!( + output + .iter() + .any(|(_, bytes)| bytes == b"warning\xc3\x28\n") + ); + assert_eq!( + read_recall_view(store, ("diagnostics", &invocation), 10) + .await + .expect("read diagnostic")[0]["message"], + json!("orders warning") + ); + assert_eq!( + read_recall_view(store, ("events", &invocation), 10) + .await + .expect("read native events") + .len(), + 1 + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read captured SQL")[0]["text"], + json!("select 7 as order_id") + ); + let delivered = read_recall_view(store, ("output_invocations", &invocation), 10) + .await + .expect("read delivery accounting"); + assert_eq!(delivered.len(), 1); + assert!( + delivered[0]["flush_complete"] == json!(1) || delivered[0]["flush_complete"] == json!(true) + ); +} + +async fn large_batches(store: &ControlStore, worker: &WorkerClient, claim_id: Uuid) { + use sqlbuild_recall::writer::models::SessionStore; + use std::time::Instant; + + let SessionStore::Hosted(opener) = worker.recall_store() else { + panic!("hosted store") + }; + tokio::task::spawn_blocking(move || { + let envelope = RowEnvelope { + invocation_id: "backlog".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + let store = opener + .open(&envelope, Instant::now() + Duration::from_secs(2)) + .expect("open native HTTP store"); + // A replacement can expand one pending lifecycle event beyond the queue's batch count. + let events = (0..501) + .map(|index| { + event_object_row(&envelope, json!({ + "event_id": format!("backlog-{index}"), "occurred_at": "2026-01-02T03:04:05Z", + "event_type": "statement_completed", "producer": "executor", "schema_version": 1, + "payload": {"row_count": index} + }).as_object().expect("event object")).expect("encode backlog event") + }) + .collect::>(); + let expanded = store.insert(&events, Instant::now() + Duration::from_secs(30)); + // Six maximum-sized native artifacts fit the 32 MiB queue, but their JSON blobs exceed 64 MiB. + let artifacts = (0..6) + .flat_map(|index| { + let content = vec![b'a' + index; 4 * 1024 * 1024]; + artifact_rows( + &envelope, + &ArtifactInput { + digest: format!("sha256:{}", sha256_hex(&content)), + content, + kind: "evidence".into(), + media_type: "application/octet-stream".into(), + seen_at: "2026-01-02T03:04:05Z".into(), + ref_id: format!("backlog-{index}"), + role: "evidence".into(), + statement_id: None, + resource_id: None, + }, + ) + }) + .collect::>(); + let bytes = store.insert(&artifacts, Instant::now() + Duration::from_secs(30)); + assert!( + expanded.is_ok() && bytes.is_ok(), + "materialized rows: {expanded:?}; wire bytes: {bytes:?}" + ); + assert_eq!( + expanded.expect("publish expanded event batch").delivered, + [(Table::Events, 501)] + ); + assert_eq!( + bytes.expect("publish large artifact batch").delivered, + [(Table::Artifacts, 6), (Table::ArtifactRefs, 6)] + ); + }) + .await + .expect("join backlog writer"); + let invocation = format!("worker/{claim_id}/backlog"); + assert_eq!( + read_recall_view(store, ("events", &invocation), 1000) + .await + .expect("read full event backlog") + .len(), + 501 + ); + assert_eq!( + read_recall_view(store, ("artifacts", &invocation), 10) + .await + .expect("read large artifacts") + .len(), + 6 + ); +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contract.rs b/crates/sqlbuild-orchestration/tests/api/test_contract.rs new file mode 100644 index 0000000000..4b311bbb1b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_contract.rs @@ -0,0 +1,581 @@ +use std::collections::BTreeMap; +use std::time::Duration; + +use crate::api::errors::ClientError; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunExecution, RunStatus}; +use crate::runs::tests::helpers::request_run; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::next_remote_generation::next_remote_generation; +use crate::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use crate::snapshots::main::verify_execution::verify_execution; +use crate::snapshots::models::{SnapshotInput, SnapshotRecord}; +use crate::store::main::migrate_store::migrate_store; +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::ApiContractTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_real_http_server_when_locations_and_workers_report_then_scopes_retries_and_terminal_expiry_are_enforced() + { + let test_cases = [ + ApiContractTestCase { + description: "SQLite worker and location HTTP contract", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + ApiContractTestCase { + description: "PostgreSQL worker and location HTTP contract", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("another location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let expiry = Utc::now() + chrono::Duration::hours(1); + let location_token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + expiry, + ) + .await + .expect("location credential"); + let api = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let path = format!( + "{}/api/locations/{location_id}/targets/development/snapshots", + api.url + ); + let response = client + .post(&path) + .header("Content-Type", "application/json") + .body("invalid-json") + .send() + .await + .expect("unauthenticated request"); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "authentication precedes JSON parsing" + ); + let openapi: serde_json::Value = client + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("OpenAPI response") + .json() + .await + .expect("OpenAPI JSON"); + assert_eq!( + openapi["components"]["securitySchemes"]["bearer"]["scheme"], + "bearer" + ); + assert!(openapi["components"]["schemas"]["SnapshotInput"].is_object()); + assert!(openapi["components"]["schemas"]["RunChangeRequest"].is_object()); + assert!(openapi["components"]["schemas"]["RunExecution"].is_object()); + assert_eq!( + next_remote_generation( + &api.url, + &location_token.bearer, + location_id, + "development" + ) + .await + .expect("first generation"), + 1 + ); + let input = snapshot(location_id); + let published = publish_remote_snapshot(&api.url, &location_token.bearer, &input) + .await + .expect("publish retained native data"); + let metadata: SnapshotRecord = client + .get(format!("{path}/metadata")) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("metadata response") + .json() + .await + .expect("metadata record without project bodies"); + assert_eq!(metadata, published); + assert_eq!( + next_remote_generation( + &api.url, + &location_token.bearer, + location_id, + "development" + ) + .await + .expect("reconcile published generation"), + 2 + ); + assert!(matches!( + next_remote_generation(&api.url, &location_token.bearer, other, "development") + .await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); + let response = client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("read published snapshot through API"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + drop(response); + assert!(matches!( + publish_remote_snapshot(&api.url, &location_token.bearer, &input).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + assert!(matches!( + publish_remote_snapshot(&api.url, &location_token.bearer, &snapshot(other)).await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); + let response = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&input) + .send() + .await + .expect("stale publish"); + assert_eq!(response.status(), StatusCode::CONFLICT); + let response = client + .post(format!( + "{}/api/locations/{other}/targets/development/snapshots", + api.url + )) + .bearer_auth(&location_token.bearer) + .json(&snapshot(other)) + .send() + .await + .expect("cross-location publish"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let current: serde_json::Value = client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("current snapshot") + .json() + .await + .expect("snapshot JSON"); + assert_eq!( + current["record"]["snapshot_id"], + published.snapshot_id.to_string() + ); + assert_eq!(current["nodes"]["orders"]["compiled_sql"], "select 1 as id"); + for image_id in [None, Some("sha256:unpublished-orders-image".into())] { + let mut mismatched = + request((location_id, published.snapshot_id), 1, 0, &["orders"]); + mismatched.image_id = image_id; + assert!(request_run(store, &mismatched).await.is_err()); + } + assert!( + list_runs(store, (location_id, "development"), 100) + .await + .expect("rejected requests do not queue") + .is_empty() + ); + queue( + store, + &request((location_id, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let run_id = claim.request.run_id; + let worker = issue_token( + store, + &audit, + CredentialScope::Run { + run_id, + claim_id: claim.claim_id, + }, + expiry, + ) + .await + .expect("worker credential"); + let events_path = format!("{}/api/runs/{run_id}/events", api.url); + let heartbeat_path = format!("{}/api/runs/{run_id}/heartbeat", api.url); + let execution_path = format!("{}/api/runs/{run_id}/execution", api.url); + let response = client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker execution contract"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["cache-control"], "no-store"); + let execution: RunExecution = response.json().await.expect("pinned execution"); + assert_eq!(execution.claim, claim); + assert_eq!(execution.environment_hash, input.environment_hash); + assert_eq!(execution.sqb_version, input.sqb_version); + assert_eq!( + execution.node_hashes.keys().collect::>(), + input.nodes.keys().collect::>(), + "dependency verification includes nodes outside the run selection" + ); + verify_execution(&input, &execution).expect("matching worker compilation"); + for field in ["target", "image_id", "environment_hash", "sqb_version"] { + let mut changed = json!(input); + changed[field] = json!("different"); + let changed: SnapshotInput = + serde_json::from_value(changed).expect("changed worker inputs"); + assert!(verify_execution(&changed, &execution).is_err(), "{field}"); + } + for field in [ + "kind", + "definition", + "raw_sql", + "compiled_sql", + "python_source", + ] { + let mut changed = json!(input); + changed["nodes"]["inventory"][field] = json!("different"); + let changed: SnapshotInput = + serde_json::from_value(changed).expect("changed dependency"); + assert!(verify_execution(&changed, &execution).is_err(), "{field}"); + } + let mut renamed = input.clone(); + let node = renamed.nodes.remove("inventory").expect("dependency"); + renamed.nodes.insert("renamed_inventory".into(), node); + assert!(verify_execution(&renamed, &execution).is_err()); + renamed.nodes.remove("renamed_inventory"); + assert!(verify_execution(&renamed, &execution).is_err()); + let mut display_only = input.clone(); + display_only.generation += 1; + display_only.git_author = Some("Example Author".into()); + display_only.git_dirty = true; + display_only + .documents + .insert("columns".into(), json!({"orders": ["id"]})); + verify_execution(&display_only, &execution) + .expect("display metadata does not change executable code"); + for (url, token) in [ + (execution_path.clone(), &location_token.bearer), + ( + format!("{}/api/runs/{}/execution", api.url, Uuid::now_v7()), + &worker.bearer, + ), + ] { + assert_eq!( + client + .get(url) + .bearer_auth(token) + .send() + .await + .expect("execution scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } + let mut redeploy = input.clone(); + redeploy.generation += 1; + redeploy.environment_hash = "updated-orders-environment".into(); + redeploy.image_id = Some("sha256:updated-orders-image".into()); + redeploy.sqb_version = "99.0.0".into(); + redeploy + .nodes + .get_mut("orders") + .expect("orders") + .compiled_sql = Some("select 2 as id".into()); + assert_eq!( + client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&redeploy) + .send() + .await + .expect("publish redeploy") + .status(), + StatusCode::OK + ); + let still_pinned: RunExecution = client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("execution after redeploy") + .json() + .await + .expect("old execution contract"); + assert_eq!(still_pinned, execution); + let started = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&location_token.bearer) + .json(&started) + .send() + .await + .expect("location cannot act as worker") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .get(&path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker cannot read location snapshot") + .status(), + StatusCode::FORBIDDEN + ); + for applied in [true, false] { + let response = client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&started) + .send() + .await + .expect("report started"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .json::() + .await + .expect("report response"), + json!({"applied": applied}) + ); + } + let wrong_claim = RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: Some(Uuid::now_v7()), + ..started.clone() + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&wrong_claim) + .send() + .await + .expect("wrong claim") + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .post(format!("{}/api/runs/{}/heartbeat", api.url, Uuid::now_v7())) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("other run") + .status(), + StatusCode::FORBIDDEN + ); + for (output_run, credential) in [ + (Uuid::now_v7(), &worker.bearer), + (run_id, &location_token.bearer), + ] { + for (endpoint, body) in [ + ("output", json!([])), + ( + "recall", + json!({"invocation_id": "execution", "records": []}), + ), + ] { + assert_eq!( + client + .post(format!("{}/api/runs/{output_run}/{endpoint}", api.url)) + .bearer_auth(credential) + .json(&body) + .send() + .await + .expect("publication scope denial") + .status(), + StatusCode::FORBIDDEN + ); + } + } + let mut forged = json!(crate::api::tests::output_helpers::chunk( + 0, + sqlbuild_recall::records::models::OutputStream::Stdout, + b"forged" + )); + forged["invocation_id"] = json!("scheduler/another-run"); + assert_eq!( + client + .post(format!("{}/api/runs/{run_id}/output", api.url)) + .bearer_auth(&worker.bearer) + .json(&json!([forged])) + .send() + .await + .expect("body cannot choose its output namespace") + .status(), + StatusCode::UNPROCESSABLE_ENTITY + ); + let cancellation = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::CancelRequested { + actor: "test-admin".into(), + reason: "Stop the orders build".into(), + }, + ..started.clone() + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&cancellation) + .send() + .await + .expect("worker cannot impersonate admin") + .status(), + StatusCode::BAD_REQUEST + ); + let response = client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("worker heartbeat"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .json::() + .await + .expect("heartbeat response"), + json!({"cancel_requested": false}) + ); + change_run(store, RunAuthority::Operator(&audit), &cancellation) + .await + .expect("operator cancellation"); + let response: serde_json::Value = client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("cancel heartbeat") + .json() + .await + .expect("cancel response"); + assert_eq!(response, json!({"cancel_requested": true})); + let terminal = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: crate::runs::models::RunOutcome::Cancelled, + reason: "worker acknowledged cancellation".into(), + exit_code: Some(143), + }, + ..started + }; + assert_eq!( + client + .post(&events_path) + .bearer_auth(&worker.bearer) + .json(&terminal) + .send() + .await + .expect("worker terminates") + .status(), + StatusCode::OK + ); + assert_eq!( + client + .post(&heartbeat_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("terminal credential rejected") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + client + .get(&execution_path) + .bearer_auth(&worker.bearer) + .send() + .await + .expect("terminal execution credential rejected") + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + list_runs(store, (location_id, "development"), 100) + .await + .expect("run projection")[0] + .status, + test_case.expected_status, + "{}", + test_case.description + ); + redeploy.generation += 1; + redeploy.image_id = None; + let local: SnapshotRecord = client + .post(&path) + .bearer_auth(&location_token.bearer) + .json(&redeploy) + .send() + .await + .expect("publish local snapshot without a container image") + .json() + .await + .expect("local snapshot response"); + let mut local_request = request((location_id, local.snapshot_id), 2, 0, &["orders"]); + local_request.image_id = None; + queue(store, &local_request).await; + revoke_token(store, &audit, location_token.identity.token_id) + .await + .expect("revoke location token"); + assert_eq!( + client + .get(&path) + .bearer_auth(&location_token.bearer) + .send() + .await + .expect("revocation effective") + .status(), + StatusCode::UNAUTHORIZED + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs new file mode 100644 index 0000000000..2faba15f57 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -0,0 +1,24 @@ +//! HTTP contracts compiled within the API owner. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "output/helpers.rs"] +mod output_helpers; +#[path = "recall/helpers.rs"] +mod recall_helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_location_health.rs"] +mod test_location_health; +#[path = "test_location_work.rs"] +mod test_location_work; +#[path = "test_reads.rs"] +mod test_reads; +#[path = "test_types.rs"] +mod test_types; +#[path = "test_warehouse_metadata.rs"] +mod test_warehouse_metadata; +#[path = "test_worker.rs"] +mod test_worker; +#[path = "work/helpers.rs"] +mod work_helpers; diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_health.rs b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs new file mode 100644 index 0000000000..002eed46e0 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_location_health.rs @@ -0,0 +1,198 @@ +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::platform::main::report_remote_location_health::report_remote_location_health; +use crate::platform::models::{LocationHealth, LocationHealthObservation, LocationHealthReport}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::LocationHealthTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_scoped_location_observations_when_retried_or_read_then_history_and_authority_are_preserved() + { + let test_cases = [ + LocationHealthTestCase { + description: "SQLite health history", + backend: Backend::Sqlite, + expected_observations: 2, + }, + LocationHealthTestCase { + description: "PostgreSQL health history", + backend: Backend::Postgres, + expected_observations: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Location { location_id }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("location token"); + let api = start_api(store).await; + let client = reqwest::Client::new(); + let url = format!("{}/api/locations/{location_id}/health", api.url); + let other_url = format!("{}/api/locations/{other}/health", api.url); + let heartbeat = LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: "preview #1".into(), + health: LocationHealth::Heartbeat {}, + }; + let before = Utc::now(); + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("register") + ); + assert!( + !report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("exact retry") + ); + let changed = LocationHealthReport { + target: "another target".into(), + ..heartbeat.clone() + }; + let response = client + .post(&url) + .bearer_auth(&token.bearer) + .json(&changed) + .send() + .await + .expect("changed retry"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let response = client + .post(&other_url) + .bearer_auth(&token.bearer) + .json(&heartbeat) + .send() + .await + .expect("foreign observation"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let response = client + .get(&other_url) + .bearer_auth(&token.bearer) + .send() + .await + .expect("foreign history"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let response = client + .post(&url) + .header("Content-Type", "application/json") + .body("invalid") + .send() + .await + .expect("anonymous report"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let response = client.post(&url).bearer_auth(&token.bearer).json(&json!({ + "event_id": Uuid::now_v7(), "session_id": heartbeat.session_id, "target": heartbeat.target, + "health": { "kind": "heartbeat", "unexpected": "must not disappear" }, + })).send().await.expect("strict health schema"); + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + let heartbeat = LocationHealthReport { + event_id: Uuid::now_v7(), + ..heartbeat + }; + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &heartbeat) + .await + .expect("refresh existing instance liveness") + ); + let connection = LocationHealthReport { + event_id: Uuid::now_v7(), + health: LocationHealth::Connection { + generation: 1, + healthy: false, + }, + ..heartbeat.clone() + }; + let response = client + .post(&url) + .bearer_auth(&token.bearer) + .json(&connection) + .send() + .await + .expect("unpublished health"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let mut input = snapshot(location_id); + input.target.clone_from(&heartbeat.target); + publish_snapshot(store, &input) + .await + .expect("published generation"); + assert!( + report_remote_location_health((&api.url, &token.bearer), location_id, &connection) + .await + .expect("connection observation") + ); + let response = client + .get(&url) + .bearer_auth(&token.bearer) + .send() + .await + .expect("health history"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let observations: Vec = + response.json().await.expect("typed health history"); + assert_eq!( + observations.len(), + test_case.expected_observations, + "{}: rejected and duplicate reports append nothing", + test_case.description + ); + assert_eq!(observations[0].report, connection); + assert_eq!(observations[1].report, heartbeat); + let events = read_recall_events(store, 0, 100) + .await + .expect("read durable events"); + assert_eq!( + events + .iter() + .filter( + |event| event.get("event_type").and_then(serde_json::Value::as_str) + == Some("location_health") + ) + .count(), + 1, + "heartbeats only update liveness; connection observations remain durable facts" + ); + assert!( + observations + .iter() + .all(|observation| observation.received_at >= before + && observation.received_at <= Utc::now()) + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_location_work.rs b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs new file mode 100644 index 0000000000..a89d419bcd --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_location_work.rs @@ -0,0 +1,304 @@ +use crate::api::errors::ClientError; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::location_work::location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::request_location_work::request_location_work; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, + LocationWorkRecord, LocationWorkRequest, LocationWorkResult, +}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use chrono::{Duration, Utc}; +use reqwest::StatusCode; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::LocationWorkTestCase; +use crate::api::tests::work_helpers::execute; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_outbound_work_when_claimed_retried_or_replaced_then_scope_and_immutable_results_are_enforced() + { + let test_cases = [ + LocationWorkTestCase { + description: "SQLite outbound work", + backend: Backend::Sqlite, + expected_generation: 1, + }, + LocationWorkTestCase { + description: "PostgreSQL outbound work", + backend: Backend::Postgres, + expected_generation: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("fresh migration"); + // Exercise both deployed predecessors, including checksums and repeated startup. + execute(store, "DROP TABLE sqb_warehouse_freshness").await; + execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DELETE FROM sqb_schema_migrations WHERE version>=3").await; + migrate_store(store).await.expect("schema 2 upgrade"); + execute(store, "DROP TABLE sqb_warehouse_freshness").await; + execute(store, "DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2").await; + migrate_store(store).await.expect("schema 1 upgrade"); + migrate_store(store).await.expect("repeat migration"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let request = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id, + target: "preview #1".into(), + reason: "Check orders connectivity".into(), + work: LocationWork::CheckConnection {}, + }; + assert!( + request_location_work(store, &audit, &request) + .await + .is_err(), + "unpublished target" + ); + let mut input = snapshot(location_id); + input.target.clone_from(&request.target); + publish_snapshot(store, &input).await.expect("publish"); + let invalid_audit = AuditContext { + granting_role: String::new(), + ..audit.clone() + }; + assert!( + request_location_work(store, &invalid_audit, &request) + .await + .is_err() + ); + assert!( + location_work(store, request.request_id) + .await + .expect("rolled back read") + .is_none() + ); + assert!( + request_location_work(store, &audit, &request) + .await + .expect("enqueue") + ); + let original = location_work(store, request.request_id) + .await + .expect("read") + .expect("request"); + input.generation = 2; + publish_snapshot(store, &input).await.expect("republish"); + assert!( + !request_location_work(store, &audit, &request) + .await + .expect("retry after redeploy") + ); + assert_eq!( + location_work(store, request.request_id) + .await + .expect("read"), + Some(original.clone()) + ); + let changed = LocationWorkRequest { + reason: "Different request".into(), + ..request.clone() + }; + assert!( + request_location_work(store, &audit, &changed) + .await + .is_err() + ); + let changed_actor = AuditContext { + actor: "another-admin".into(), + ..audit.clone() + }; + assert!( + request_location_work(store, &changed_actor, &request) + .await + .is_err() + ); + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + Utc::now() + Duration::hours(1), + ) + .await + .expect("token"); + let api = start_api(store).await; + let worker = + connect_location_work((&api.url, &token.bearer), location_id).expect("work client"); + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: request.target.clone(), + }; + assert!(worker.poll(&poll).await.is_err(), "register before polling"); + for session_id in [poll.session_id, Uuid::nil()] { + report_location_health( + store, + location_id, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register instance"); + } + let wrong_target = LocationWorkPoll { + target: "another target".into(), + ..poll.clone() + }; + assert!(worker.poll(&wrong_target).await.is_err()); + let foreign = + connect_location_work((&api.url, &token.bearer), other).expect("foreign client"); + assert!(matches!( + foreign.poll(&poll).await, + Err(ClientError::Rejected(StatusCode::FORBIDDEN)) + )); + let client = reqwest::Client::new(); + let url = format!("{}/api/locations/{location_id}/work", api.url); + let response = client + .post(format!("{url}/poll")) + .header("Content-Type", "application/json") + .body("invalid") + .send() + .await + .expect("anonymous"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let claim = worker.poll(&poll).await.expect("poll").expect("claim"); + assert_eq!( + claim.generation, test_case.expected_generation, + "{}: request remains pinned despite redeployment", + test_case.description + ); + assert_eq!(claim.request, request); + assert_eq!( + worker.poll(&poll).await.expect("lost poll response retry"), + Some(claim.clone()) + ); + let replacement = LocationWorkPoll { + session_id: Uuid::nil(), + ..poll.clone() + }; + assert!( + worker + .poll(&replacement) + .await + .expect("another instance") + .is_none() + ); + let stale = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: false }, + }; + execute( + store, + "UPDATE sqb_location_work SET claimed_until='2000-01-01T00:00:00Z'", + ) + .await; + assert!(matches!( + worker.complete(request.request_id, &stale).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + let reclaimed = worker + .poll(&replacement) + .await + .expect("reclaim") + .expect("new claim"); + assert_ne!(claim.claim_id, reclaimed.claim_id); + assert!(matches!( + worker.complete(request.request_id, &stale).await, + Err(ClientError::Rejected(StatusCode::CONFLICT)) + )); + let result = LocationWorkResult { + claim_id: reclaimed.claim_id, + session_id: replacement.session_id, + outcome: LocationWorkOutcome::Superseded {}, + }; + assert!( + worker + .complete(request.request_id, &result) + .await + .expect("complete") + ); + execute(store, "UPDATE sqb_location_work SET expires_at='2000-01-01T00:00:00Z', claimed_until='2000-01-01T00:00:00Z'").await; + assert!( + !worker + .complete(request.request_id, &result) + .await + .expect("lost acknowledgement retry after expiry") + ); + let changed_result = LocationWorkResult { + outcome: LocationWorkOutcome::Connection { healthy: true }, + ..result.clone() + }; + assert!( + worker + .complete(request.request_id, &changed_result) + .await + .is_err() + ); + assert!( + worker + .poll(&poll) + .await + .expect("completed excluded") + .is_none() + ); + let response = client + .get(format!("{url}/{}", request.request_id)) + .bearer_auth(&token.bearer) + .send() + .await + .expect("result read"); + assert_eq!(response.headers()["cache-control"], "no-store"); + let record: LocationWorkRecord = response.json().await.expect("typed result"); + assert_eq!(record.result_json, Some(result)); + assert!(record.completed_at.is_some()); + let next = LocationWorkRequest { + request_id: Uuid::now_v7(), + ..request + }; + request_location_work(store, &audit, &next) + .await + .expect("another request"); + execute( + store, + "UPDATE sqb_location_work SET expires_at='2000-01-01T00:00:00Z'", + ) + .await; + assert!( + worker + .poll(&poll) + .await + .expect("expired excluded") + .is_none() + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_reads.rs b/crates/sqlbuild-orchestration/tests/api/test_reads.rs new file mode 100644 index 0000000000..1a3b7a06b9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_reads.rs @@ -0,0 +1,522 @@ +//! Real HTTP reads keep both location and target scope, including guessed nested resource IDs. + +use std::collections::BTreeMap; +use std::time::Duration; + +use crate::credentials::main::create_principal::create_principal; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome}; +use crate::runs::tests::helpers::request_run; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::publish_recall; +use crate::worker::main::connect_worker::connect_worker; +use chrono::Utc; +use reqwest::StatusCode; +use serde_json::json; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{OutputStream, RowEnvelope}; +use uuid::Uuid; + +use crate::api::tests::helpers::{read_json, start_api, stop_api}; +use crate::api::tests::output_helpers::chunk; +use crate::api::tests::recall_helpers::recall_contract; +use crate::api::tests::test_types::ApiReadTestCase; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_published_locations_and_scheduled_work_when_read_through_http_then_resources_and_histories_remain_scoped() + { + let test_cases = [ + ApiReadTestCase { + description: "SQLite scoped HTTP reads", + backend: Backend::Sqlite, + expected_runs: 3, + expected_ticks: 3, + }, + ApiReadTestCase { + description: "PostgreSQL scoped HTTP reads", + backend: Backend::Postgres, + expected_runs: 3, + expected_ticks: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", "2020-01-03T00:00:00Z", "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}})); + let published = publish_snapshot(store, &input) + .await + .expect("publish declarations"); + let trigger_id = list_triggers(store, location_id, "development") + .await + .expect("trigger")[0] + .trigger_id; + let mut foreign = input.clone(); + foreign.location_id = other; + let foreign_snapshot = publish_snapshot(store, &foreign) + .await + .expect("publish other location"); + let foreign_trigger = list_triggers(store, other, "development") + .await + .expect("foreign trigger")[0] + .trigger_id; + let foreign_run = request( + (other, foreign_snapshot.snapshot_id), + 1001, + 0, + &["inventory"], + ); + request_run(store, &foreign_run) + .await + .expect("queue foreign run"); + let mut another_target = input.clone(); + another_target.target = "validation".into(); + publish_snapshot(store, &another_target) + .await + .expect("publish second target"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("evaluate due schedule"); + let mut manual_ids = Vec::new(); + for (id, node) in [(100, "customers"), (101, "products")] { + let run = request((location_id, published.snapshot_id), id, 0, &[node]); + manual_ids.push(run.run_id); + request_run(store, &run).await.expect("queue manual run"); + } + let change = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: "test-admin".into(), + reason: "Pause orders refresh".into(), + change: TriggerChange::Pause, + }; + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + change_trigger(store, &audit, &change) + .await + .expect("pause trigger"); + let expiry = Utc::now() + chrono::Duration::hours(1); + let token = issue_token( + store, + &audit, + CredentialScope::Location { location_id }, + expiry, + ) + .await + .expect("location token"); + let principal_id = + create_principal(store, &audit, ("orders operator", PrincipalKind::Person)) + .await + .expect("unprivileged person"); + let principal = issue_token( + store, + &audit, + CredentialScope::Principal { principal_id }, + expiry, + ) + .await + .expect("principal token"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location_id, "development")), + ) + .await + .expect("claim") + .expect("work"); + let worker = issue_token( + store, + &audit, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + expiry, + ) + .await + .expect("worker token"); + let api = start_api(store).await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("HTTP client"); + let prefix = format!( + "{}/api/locations/{location_id}/targets/development", + api.url + ); + let run_path = format!("{prefix}/runs"); + let node_path = format!("{prefix}/runs/{}/nodes", manual_ids[0]); + let trigger_path = format!("{prefix}/triggers"); + let event_path = format!("{trigger_path}/{trigger_id}/events"); + let tick_path = format!("{trigger_path}/{trigger_id}/ticks"); + let invocation_path = format!("{prefix}/runs/{}/invocations", claim.request.run_id); + let recall_path = format!( + "{prefix}/runs/{}/recall/events?invocation=worker/{}/execution", + claim.request.run_id, claim.claim_id + ); + let paths = [ + &run_path, + &node_path, + &trigger_path, + &event_path, + &tick_path, + &invocation_path, + &recall_path, + ]; + let connection = connect_worker( + &api.url, + &worker.bearer, + (claim.request.run_id, claim.claim_id), + ) + .expect("worker HTTP connection"); + recall_contract(store, &connection, claim.claim_id).await; + let output = chunk(0, OutputStream::Stdout, b"orders worker output"); + connection + .publish_output(std::slice::from_ref(&output)) + .await + .expect("output-only namespace"); + let neighboring = RowEnvelope { + invocation_id: format!("worker/{}-other", claim.claim_id), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + publish_recall(store, &[output_row(&neighboring, &output)]) + .await + .expect("neighboring namespace"); + let invocations = read_json(&client, (&invocation_path, &token.bearer)).await; + let invocation_ids = invocations + .as_array() + .expect("run invocations") + .iter() + .map(|row| { + row["invocation_id"] + .as_str() + .expect("invocation ID") + .to_owned() + }) + .collect::>(); + assert_eq!( + invocation_ids, + vec![ + format!("scheduler/{}", claim.request.run_id), + format!("worker/{}", claim.claim_id), + format!("worker/{}/another-invocation", claim.claim_id), + format!("worker/{}/backlog", claim.claim_id), + format!("worker/{}/execution", claim.claim_id), + format!("worker/{}/native-writer", claim.claim_id), + ] + ); + let evidence = read_json(&client, (&recall_path, &token.bearer)).await; + assert_eq!(evidence.as_array().expect("worker events").len(), 1); + assert_eq!(evidence[0]["duration_ms"], json!(12.75)); + assert_eq!(evidence[0]["row_count"], json!(9)); + assert_eq!( + read_json(&client, (&format!("{recall_path}&limit=0"), &token.bearer)).await, + json!([]) + ); + for path in [ + recall_path.replace( + &format!("worker/{}/execution", claim.claim_id), + &format!("scheduler/{}", foreign_run.run_id), + ), + recall_path.replace( + &format!("worker/{}/execution", claim.claim_id), + &format!("worker/{}-other/execution", claim.claim_id), + ), + recall_path.replace( + &format!("/runs/{}/", claim.request.run_id), + &format!("/runs/{}/", manual_ids[0]), + ), + recall_path.replace( + &format!("/runs/{}/", claim.request.run_id), + &format!("/runs/{}/", foreign_run.run_id), + ), + recall_path.replace("/development/", "/validation/"), + ] { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("Recall scope denial") + .status(), + StatusCode::NOT_FOUND + ); + } + assert_eq!( + client + .get(recall_path.replace("/recall/events?", "/recall/unknown?")) + .bearer_auth(&token.bearer) + .send() + .await + .expect("unknown view") + .status(), + StatusCode::BAD_REQUEST + ); + let scheduler_path = format!( + "{prefix}/runs/{}/recall/events?invocation=scheduler/{}", + claim.request.run_id, claim.request.run_id + ); + let history = read_json(&client, (&scheduler_path, &token.bearer)).await; + assert!( + history + .as_array() + .expect("run history") + .iter() + .any(|row| row["event_type"] == "run_claimed") + ); + let runs = read_json(&client, (&run_path, &token.bearer)).await; + assert_eq!( + runs.as_array().expect("runs").len(), + test_case.expected_runs, + "{}", + test_case.description + ); + assert!( + runs.as_array() + .expect("runs") + .iter() + .all(|run| run["location_id"] == location_id.to_string() + && run["target"] == "development") + ); + let nodes = read_json(&client, (&node_path, &token.bearer)).await; + assert_eq!(nodes.as_array().expect("nodes").len(), 1); + assert_eq!(nodes[0]["node_key"], "customers"); + assert_eq!(nodes[0]["status"], "pending"); + let triggers = read_json(&client, (&trigger_path, &token.bearer)).await; + assert_eq!(triggers.as_array().expect("triggers").len(), 1); + assert_eq!(triggers[0]["trigger_id"], trigger_id.to_string()); + assert_eq!(triggers[0]["status"], "paused"); + let events = read_json(&client, (&event_path, &token.bearer)).await; + assert_eq!(events.as_array().expect("events").len(), 1); + assert_eq!(events[0]["request_json"], json!(change)); + let ticks = read_json(&client, (&tick_path, &token.bearer)).await; + assert_eq!( + ticks.as_array().expect("ticks").len(), + test_case.expected_ticks + ); + assert_eq!(ticks[0]["outcome"], "requested"); + assert_eq!(ticks[0]["scheduled_for"], "2020-01-03T00:00:00Z"); + assert_eq!(ticks[1]["outcome"], "covered"); + for path in [&run_path, &event_path, &tick_path, &invocation_path] { + assert_eq!( + read_json(&client, (&format!("{path}?limit=1"), &token.bearer)) + .await + .as_array() + .expect("bounded history") + .len(), + 1 + ); + assert_eq!( + read_json(&client, (&format!("{path}?limit=0"), &token.bearer)).await, + json!([]) + ); + assert_eq!( + client + .get(format!("{path}?limit=invalid")) + .bearer_auth(&token.bearer) + .send() + .await + .expect("invalid limit") + .status(), + StatusCode::BAD_REQUEST + ); + } + for path in paths { + assert_eq!( + client + .get(path) + .send() + .await + .expect("unauthenticated read") + .status(), + StatusCode::UNAUTHORIZED + ); + for bearer in [&principal.bearer, &worker.bearer] { + assert_eq!( + client + .get(path) + .bearer_auth(bearer) + .send() + .await + .expect("wrong credential scope") + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + client + .get(path.replace(&location_id.to_string(), &other.to_string())) + .bearer_auth(&token.bearer) + .send() + .await + .expect("cross-location read") + .status(), + StatusCode::FORBIDDEN + ); + } + for path in [ + node_path.replace(&manual_ids[0].to_string(), &foreign_run.run_id.to_string()), + event_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + tick_path.replace(&trigger_id.to_string(), &foreign_trigger.to_string()), + invocation_path.replace( + &claim.request.run_id.to_string(), + &foreign_run.run_id.to_string(), + ), + node_path.replace("/development/", "/validation/"), + event_path.replace("/development/", "/validation/"), + tick_path.replace("/development/", "/validation/"), + invocation_path.replace("/development/", "/validation/"), + ] { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("nested ID cannot bypass scope") + .status(), + StatusCode::NOT_FOUND + ); + } + assert_eq!( + read_json( + &client, + ( + &run_path.replace("/development/", "/validation/"), + &token.bearer + ) + ) + .await, + json!([]) + ); + let validation = read_json( + &client, + ( + &trigger_path.replace("/development/", "/validation/"), + &token.bearer, + ), + ) + .await; + assert_eq!(validation.as_array().expect("other target").len(), 1); + assert_ne!(validation[0]["trigger_id"], trigger_id.to_string()); + let openapi: serde_json::Value = client + .get(format!("{}/api/openapi.json", api.url)) + .send() + .await + .expect("OpenAPI") + .json() + .await + .expect("OpenAPI JSON"); + for path in [ + "/api/locations/{location_id}/targets/{target}/runs", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/invocations", + "/api/locations/{location_id}/targets/{target}/runs/{run_id}/recall/{view}", + "/api/locations/{location_id}/targets/{target}/triggers", + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/events", + "/api/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks", + ] { + assert_eq!( + openapi["paths"][path]["get"]["security"], + json!([{"bearer": []}]) + ); + } + input.generation += 1; + input.documents.insert("triggers".into(), json!({})); + publish_snapshot(store, &input) + .await + .expect("remove declaration"); + assert_eq!( + read_json(&client, (&trigger_path, &token.bearer)).await, + json!([]) + ); + assert_eq!( + read_json(&client, (&event_path, &token.bearer)).await, + events + ); + assert_eq!(read_json(&client, (&tick_path, &token.bearer)).await, ticks); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "Stop scoped read fixture".into(), + exit_code: Some(1), + }, + }, + ) + .await + .expect("terminalize worker and release liveness"); + assert_eq!( + read_json(&client, (&invocation_path, &token.bearer)).await, + invocations, + "invocation discovery uses retained facts, not active liveness" + ); + assert_eq!( + read_json(&client, (&recall_path, &token.bearer)).await, + evidence, + "historical Recall remains authorized after the active claim is removed" + ); + revoke_token(store, &audit, token.identity.token_id) + .await + .expect("revoke location"); + for path in paths { + assert_eq!( + client + .get(path) + .bearer_auth(&token.bearer) + .send() + .await + .expect("revoked read") + .status(), + StatusCode::UNAUTHORIZED + ); + } + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs new file mode 100644 index 0000000000..71c81b400d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -0,0 +1,52 @@ +use crate::runs::models::RunStatus; +use crate::store::errors::ControlError; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct ApiContractTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: RunStatus, +} + +pub(crate) struct WorkerRedirectTestCase { + pub(crate) description: &'static str, + pub(crate) expected_status: reqwest::StatusCode, +} + +pub(crate) struct WorkerDeadlineTestCase { + pub(crate) description: &'static str, + pub(crate) budget: std::time::Duration, + pub(crate) expected_kind: sqlbuild_recall::store::errors::StoreErrorKind, + pub(crate) expected_arrival: bool, +} + +pub(crate) struct ApiReadTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, + pub(crate) expected_ticks: usize, +} + +pub(crate) struct TestApi { + pub(crate) url: String, + pub(crate) shutdown: tokio::sync::oneshot::Sender<()>, + pub(crate) task: tokio::task::JoinHandle>, +} + +pub(crate) struct LocationHealthTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_observations: usize, +} + +pub(crate) struct LocationWorkTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_generation: i64, +} + +pub(crate) struct WarehouseMetadataTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_observations: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs new file mode 100644 index 0000000000..efc31663b5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -0,0 +1,306 @@ +//! A scoped HTTP reader mirrors immutable warehouse observations, including external runs. + +use chrono::{Duration, Utc}; +use reqwest::StatusCode; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::test_types::WarehouseMetadataTestCase; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::TransactionMode; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::models::SourceFreshnessBatch; + +#[test] +fn given_warehouse_observations_when_pages_retry_then_scope_history_and_atomicity_are_preserved() { + let test_cases = [ + WarehouseMetadataTestCase { + description: "SQLite warehouse history", + backend: Backend::Sqlite, + expected_observations: 4, + }, + WarehouseMetadataTestCase { + description: "PostgreSQL warehouse history", + backend: Backend::Postgres, + expected_observations: 4, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let input = snapshot(location); + publish_snapshot(store, &input).await.expect("snapshot"); + let session_id = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id, + target: "development".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("registered reader"); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Location { + location_id: location, + }, + Utc::now() + Duration::hours(1), + ) + .await + .expect("token"); + let api = start_api(store).await; + let client = reqwest::Client::new(); + let url = format!( + "{}/api/locations/{location}/targets/development/metadata/source-freshness", + api.url + ); + let record = json!({ + "event_id": "warehouse-observation-1", + "state_database": null, "state_schema": "main", "source_name": "raw_orders", + "target_database": null, "target_schema": "main", "target_name": "orders", + "run_id": "external-command-1", "strategy": "max_column", "value_kind": "integer", + "data_version": "18446744073709551617", "data_version_hash": "same-data-version", + "observed_at": "2026-01-01T10:00:00Z", + }); + let mut later = record.clone(); + later["event_id"] = json!("warehouse-observation-2"); + later["observed_at"] = json!("2026-01-01T11:00:00Z"); + let mut other_run = record.clone(); + other_run + .as_object_mut() + .expect("record") + .remove("event_id"); + other_run["run_id"] = json!("external-command-2"); + let mut distinct = record.clone(); + distinct["event_id"] = json!("warehouse-observation-3"); + let mut page = vec![record.clone(); 100]; + page[1] = later; + page[2] = other_run.clone(); + page[3] = distinct; + // Both missing and explicitly NULL identities use the same fixed legacy fallback. + other_run["event_id"] = json!(null); + page[4] = other_run; + let batch = json!({"session_id": session_id, "generation": 1, "records": page}); + let outbound = connect_location_work((&api.url, &token.bearer), location) + .expect("pooled native location client"); + let typed: SourceFreshnessBatch = serde_json::from_value(batch.clone()).expect("page"); + for expected_applied in [true, false] { + assert_eq!( + outbound + .sync_source_freshness("development", &typed) + .await + .expect("native client sync"), + expected_applied + ); + } + let mut equivalent = batch.clone(); + equivalent["records"] = json!([record]); + equivalent["records"][0]["observed_at"] = json!("2026-01-01T11:00:00+01:00"); + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&equivalent) + .send() + .await + .expect("equivalent timestamp") + .json::() + .await + .expect("reply"), + json!({"applied": false}) + ); + let mut stale = batch.clone(); + stale["generation"] = json!(2); + let mut invalid = batch.clone(); + invalid["records"][1]["source_name"] = json!(" "); + let mut invalid_id = batch.clone(); + invalid_id["records"][0]["event_id"] = json!(" "); + let mut oversized = batch.clone(); + oversized["records"] = json!(vec![record.clone(); 101]); + let mut unknown = batch.clone(); + unknown["records"][0]["status"] = json!("success"); + let foreign = url.replace(&location.to_string(), &other.to_string()); + let other_target = url.replace("/development/", "/production/"); + for (endpoint, body, expected) in [ + (&url, stale, StatusCode::CONFLICT), + (&url, invalid, StatusCode::BAD_REQUEST), + (&url, invalid_id, StatusCode::BAD_REQUEST), + (&url, oversized, StatusCode::BAD_REQUEST), + (&url, unknown, StatusCode::UNPROCESSABLE_ENTITY), + (&foreign, batch.clone(), StatusCode::FORBIDDEN), + (&other_target, batch.clone(), StatusCode::BAD_REQUEST), + ] { + assert_eq!( + client + .post(endpoint) + .bearer_auth(&token.bearer) + .json(&body) + .send() + .await + .expect("rejected page") + .status(), + expected + ); + } + assert_eq!( + client + .post(&url) + .body("invalid") + .send() + .await + .expect("anonymous") + .status(), + StatusCode::UNAUTHORIZED + ); + let fault = TransactionFault { + description: "warehouse page commits all records or none", + table: "sqb_warehouse_freshness", + operation: "INSERT", + condition: "WHEN NEW.record_json->>'source_name'='inventory'", + sqlite_body: "SELECT RAISE(ABORT, 'injected metadata failure')", + postgres_body: "RAISE EXCEPTION 'injected metadata failure'", + }; + install_fault(store, &fault).await; + let mut new_record = record.clone(); + new_record["state_schema"] = json!("reporting"); + let mut inventory = new_record.clone(); + inventory["event_id"] = json!("inventory-observation-1"); + inventory["source_name"] = json!("inventory"); + let mut additional = batch.clone(); + additional["records"] = json!([new_record, inventory]); + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&additional) + .send() + .await + .expect("faulted page") + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + remove_fault(store, &fault).await; + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT record_json FROM sqb_warehouse_freshness ORDER BY observed_at,event_id", + vec![], + ) + .await + .expect("mirrored history"); + commit(transaction).await.expect("close read"); + assert_eq!( + rows.len(), + test_case.expected_observations, + "{}: no observation collapsed by data version and no partial page committed", + test_case.description + ); + for row in rows { + assert_eq!(row["record_json"]["data_version"], "18446744073709551617"); + assert_eq!(row["record_json"]["state_schema"], "main"); + } + assert_eq!( + client + .post(&url) + .bearer_auth(&token.bearer) + .json(&additional) + .send() + .await + .expect("retry after rollback") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + migrate_store(store) + .await + .expect("warm migration preserves observations"); + let (transaction, rows) = query( + begin(store, TransactionMode::Read).await.expect("read"), + "SELECT COUNT(*) AS n FROM sqb_warehouse_freshness", + vec![], + ) + .await + .expect("retained history"); + commit(transaction).await.expect("close read"); + assert_eq!(rows[0]["n"], 6); + // The same warehouse rows in another target are separate observations, not replays. + let preview_session = Uuid::now_v7(); + let mut preview = input.clone(); + preview.target = "preview".into(); + publish_snapshot(store, &preview) + .await + .expect("preview snapshot"); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: preview_session, + target: "preview".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("preview reader"); + let mut preview_batch = batch.clone(); + preview_batch["session_id"] = json!(preview_session); + let preview_url = url.replace("/development/", "/preview/"); + assert_eq!( + client + .post(&preview_url) + .bearer_auth(&token.bearer) + .json(&preview_batch) + .send() + .await + .expect("separate target") + .json::() + .await + .expect("reply"), + json!({"applied": true}) + ); + let (transaction, rows) = query(begin(store, TransactionMode::Read).await.expect("read"), + "SELECT target, COUNT(*) AS n FROM sqb_warehouse_freshness GROUP BY target ORDER BY target", vec![]).await.expect("isolated history"); + commit(transaction).await.expect("close read"); + assert_eq!( + serde_json::to_value(rows).expect("rows"), + json!([{"target": "development", "n": 6}, {"target": "preview", "n": 4}]) + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_worker.rs b/crates/sqlbuild-orchestration/tests/api/test_worker.rs new file mode 100644 index 0000000000..e9d9e1aa80 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_worker.rs @@ -0,0 +1,478 @@ +use std::collections::BTreeMap; + +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::worker::errors::WorkerError; +use crate::worker::main::connect_worker::connect_worker; +use crate::worker::main::start_worker::start_worker; +use axum::Router; +use axum::response::Redirect; +use axum::routing::get; +use chrono::Utc; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::api::tests::output_helpers::{chunk, output_contract}; +use crate::api::tests::recall_helpers::recall_contract; +use crate::api::tests::test_types::{ + ApiContractTestCase, TestApi, WorkerDeadlineTestCase, WorkerRedirectTestCase, +}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_claimed_run_when_a_worker_connects_then_verification_precedes_start_and_reports_are_retryable() + { + let test_cases = [ + ApiContractTestCase { + description: "SQLite worker HTTP lifecycle", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + ApiContractTestCase { + description: "PostgreSQL worker HTTP lifecycle", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let baseline = snapshot(location); + let publication = publish_snapshot(store, &baseline).await.expect("publish"); + queue( + store, + &request((location, publication.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let identity = (claim.request.run_id, claim.claim_id); + let token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Run { + run_id: identity.0, + claim_id: identity.1, + }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("worker token"); + let api = start_api(store).await; + let worker = connect_worker(&api.url, &token.bearer, identity).expect("worker client"); + assert!( + !format!("{worker:?}").contains(&token.bearer), + "{}", + test_case.description + ); + let wrong_claim = connect_worker(&api.url, &token.bearer, (identity.0, Uuid::now_v7())) + .expect("different local claim"); + assert!( + matches!( + wrong_claim.execution().await, + Err(WorkerError::Configuration(_)) + ), + "{}", + test_case.description + ); + let mut drifted = baseline.clone(); + drifted + .nodes + .get_mut("inventory") + .expect("unselected dependency") + .raw_sql = Some("select 2 as id".into()); + assert!( + matches!( + start_worker(&worker, &drifted).await, + Err(WorkerError::Verification(_)) + ), + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("unstarted"); + assert_eq!( + runs[0].status, + RunStatus::Queued, + "{}", + test_case.description + ); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: identity.0, + claim_id: Some(identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("record process launch"); + let session = start_worker(&worker, &baseline) + .await + .expect("verified start"); + output_contract(store, &worker, identity.1).await; + let recall_batch = recall_contract(store, &worker, identity.1).await; + assert_eq!(session.execution.claim, claim, "{}", test_case.description); + assert_eq!( + start_worker(&worker, &baseline) + .await + .expect("retry start") + .execution, + session.execution, + "{}", + test_case.description + ); + assert!( + !worker.heartbeat().await.expect("heartbeat"), + "{}", + test_case.description + ); + let started = Uuid::now_v7(); + assert!( + worker + .report( + started, + RunChange::NodeStarted { + node_key: "orders".into() + } + ) + .await + .expect("node start"), + "{}", + test_case.description + ); + assert!( + !worker + .report( + started, + RunChange::NodeStarted { + node_key: "orders".into() + } + ) + .await + .expect("retry node start"), + "{}", + test_case.description + ); + worker + .report( + Uuid::now_v7(), + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(7), + error_code: None, + skipped_because_of: None, + }, + ) + .await + .expect("node outcome"); + worker + .report( + Uuid::now_v7(), + RunChange::Finished { + outcome: RunOutcome::Success, + reason: "warehouse work completed".into(), + exit_code: Some(0), + }, + ) + .await + .expect("finish"); + assert!(matches!( + worker.publish_recall(&recall_batch).await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); + assert!(matches!( + worker + .publish_output(&[chunk( + 5, + sqlbuild_recall::records::models::OutputStream::Stdout, + b"late" + )]) + .await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + )); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("completed"); + assert_eq!( + runs[0].status, test_case.expected_status, + "{}", + test_case.description + ); + assert!( + matches!( + worker.heartbeat().await, + Err(WorkerError::Rejected(reqwest::StatusCode::UNAUTHORIZED)) + ), + "{}", + test_case.description + ); + queue( + store, + &request((location, publication.snapshot_id), 2, 0, &["orders"]), + ) + .await; + let cancelled_claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim another run") + .expect("queued work"); + let cancelled_identity = (cancelled_claim.request.run_id, cancelled_claim.claim_id); + let cancelled_token = issue_token( + store, + &AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + CredentialScope::Run { + run_id: cancelled_identity.0, + claim_id: cancelled_identity.1, + }, + Utc::now() + chrono::Duration::hours(1), + ) + .await + .expect("another worker credential"); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: cancelled_identity.0, + claim_id: Some(cancelled_identity.1), + change: RunChange::Launched { container_id: None }, + }, + ) + .await + .expect("another launched process"); + change_run( + store, + RunAuthority::Operator(&AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: cancelled_identity.0, + claim_id: Some(cancelled_identity.1), + change: RunChange::CancelRequested { + actor: "test-operator".into(), + reason: "Stop before execution".into(), + }, + }, + ) + .await + .expect("cancel before execution"); + let cancelled_worker = + connect_worker(&api.url, &cancelled_token.bearer, cancelled_identity) + .expect("cancelled worker connection"); + assert!( + matches!( + start_worker(&cancelled_worker, &baseline).await, + Err(WorkerError::Cancelled) + ), + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 10) + .await + .expect("cancelled run has not started"); + let cancelled_run = runs + .iter() + .find(|run| run.run_id == cancelled_identity.0) + .expect("cancelled projection"); + assert_eq!( + cancelled_run.status, + RunStatus::Launched, + "{}", + test_case.description + ); + assert!( + cancelled_run.started_at.is_none(), + "{}", + test_case.description + ); + drop(session); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_credentialed_worker_when_the_server_redirects_then_the_redirect_is_not_followed() { + let test_cases = [WorkerRedirectTestCase { + description: "a run bearer must not be forwarded by an API redirect", + expected_status: reqwest::StatusCode::TEMPORARY_REDIRECT, + }]; + for test_case in test_cases { + run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("isolated redirect server"); + let url = format!("http://{}", listener.local_addr().expect("address")); + let application = Router::new().route( + "/api/runs/{run_id}/execution", + get(|| async { Redirect::temporary("/must-not-receive-a-credential") }), + ); + let (shutdown, signal) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, application) + .with_graceful_shutdown(async { + let _ = signal.await; + }) + .await + .map_err(ControlError::from) + }); + let api = TestApi { + url, + shutdown, + task, + }; + let client = connect_worker( + &api.url, + "synthetic-worker-bearer", + (Uuid::now_v7(), Uuid::now_v7()), + ) + .expect("worker client"); + let Err(WorkerError::Rejected(status)) = client.execution().await else { + panic!("{}", test_case.description); + }; + assert_eq!( + status, test_case.expected_status, + "{}", + test_case.description + ); + stop_api(api).await; + }); + } +} + +#[test] +fn given_a_slow_recall_server_when_the_delivery_deadline_expires_then_the_native_store_stops_waiting() + { + use sqlbuild_recall::records::models::RowEnvelope; + use sqlbuild_recall::store::errors::StoreErrorKind; + use sqlbuild_recall::writer::models::SessionStore; + use std::time::{Duration, Instant}; + + let test_cases = [ + WorkerDeadlineTestCase { + description: "HTTP wait expires within the close budget", + budget: Duration::from_millis(50), + expected_kind: StoreErrorKind::Busy, + expected_arrival: true, + }, + WorkerDeadlineTestCase { + description: "expired budget sends no request", + budget: Duration::ZERO, + expected_kind: StoreErrorKind::Busy, + expected_arrival: false, + }, + ]; + for test_case in test_cases { + run(async { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind slow server"); + let url = format!( + "http://{}", + listener.local_addr().expect("slow server address") + ); + let arrived = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let observed = arrived.clone(); + let application = Router::new().route( + "/api/runs/{run_id}/recall", + axum::routing::post(move || { + let arrived = arrived.clone(); + async move { + arrived.store(true, std::sync::atomic::Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(2)).await; + axum::Json(sqlbuild_recall::store::models::InsertOutcome::default()) + } + }), + ); + let task = tokio::spawn(async move { axum::serve(listener, application).await }); + let client = connect_worker( + &url, + "synthetic-worker-bearer", + (Uuid::now_v7(), Uuid::now_v7()), + ) + .expect("connect worker to slow server"); + let SessionStore::Hosted(opener) = client.recall_store() else { + panic!("hosted store") + }; + tokio::task::spawn_blocking(move || { + let envelope = RowEnvelope { + invocation_id: "deadline".into(), + invocation_started_at: "2026-01-02T03:04:05Z".into(), + producer_version: "test".into(), + }; + let store = opener + .open(&envelope, Instant::now() + Duration::from_secs(1)) + .expect("open HTTP store"); + let started = Instant::now(); + let error = store + .insert(&[], started + test_case.budget) + .expect_err("deadline rejects delivery"); + assert_eq!( + error.kind, test_case.expected_kind, + "{}", + test_case.description + ); + assert!( + started.elapsed() < Duration::from_secs(1), + "{}", + test_case.description + ); + }) + .await + .expect("join slow-server publication"); + assert_eq!( + observed.load(std::sync::atomic::Ordering::SeqCst), + test_case.expected_arrival, + "{}", + test_case.description + ); + task.abort(); + let _ = task.await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/work/helpers.rs b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs new file mode 100644 index 0000000000..2debfcd9b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/work/helpers.rs @@ -0,0 +1,20 @@ +//! Fault injection changes only disposable test stores, never product time or lease semantics. + +use crate::store::models::ControlStore; + +pub(crate) async fn execute(store: &ControlStore, sql: &str) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!("SQLite store") + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(sql) + .expect("inject SQLite fault"); + return; + }; + sqlx_core::raw_sql::raw_sql(sql) + .execute(pool) + .await + .expect("inject PostgreSQL fault"); +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs new file mode 100644 index 0000000000..4c28e90101 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/helpers.rs @@ -0,0 +1,39 @@ +use crate::store::models::ControlStore; + +pub(crate) async fn location_state(store: &ControlStore) -> (i64, i64) { + let sql = "SELECT (SELECT COUNT(*) FROM sqb_locations), \ + (SELECT COUNT(*) FROM sqb_security_audit WHERE action='location.created')"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect locations") + .query_row(sql, [], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("location and audit counts"); + }; + sqlx_core::query_as::query_as(sql) + .fetch_one(pool) + .await + .expect("location and audit counts") +} + +pub(crate) async fn stored_hashes(store: &ControlStore) -> Vec { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + let connection = rusqlite::Connection::open(path).expect("inspect durable tokens"); + return connection + .prepare("SELECT secret_hash FROM sqb_access_tokens") + .expect("token hashes") + .query_map([], |row| row.get(0)) + .expect("read token hashes") + .collect::>() + .expect("hashes"); + }; + sqlx_core::query_scalar::query_scalar("SELECT secret_hash FROM sqb_access_tokens") + .fetch_all(pool) + .await + .expect("read durable token hashes") +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs new file mode 100644 index 0000000000..9a48e51758 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/locations/helpers.rs @@ -0,0 +1,24 @@ +//! Shared location fixtures use the same audited identity operation as operator bootstrap. + +use crate::credentials::models::AuditContext; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn create_location( + store: &ControlStore, + name: &str, + actor: &str, +) -> Result { + crate::credentials::main::create_location::create_location( + store, + name, + &AuditContext { + actor: actor.into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + ) + .await +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs new file mode 100644 index 0000000000..d26aa25639 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contract.rs @@ -0,0 +1,239 @@ +use std::collections::BTreeMap; + +use crate::credentials::main::authenticate_token::authenticate_token; +use crate::credentials::main::create_location::create_location; +use crate::credentials::main::create_principal::create_principal; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::main::revoke_token::revoke_token; +use crate::credentials::models::{AuditContext, CredentialScope, PrincipalKind}; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use chrono::{Duration, Utc}; +use uuid::Uuid; + +use crate::credentials::tests::helpers::{location_state, stored_hashes}; +use crate::credentials::tests::test_types::{CredentialTestCase, TransactionFault}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::terminal_helpers::finish; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_scoped_tokens_when_revoked_or_the_run_ends_then_all_replicas_reject_them_and_audit_failure_rolls_back() + { + let test_cases = [ + CredentialTestCase { + description: "SQLite credential lifecycle and atomic audit", + backend: Backend::Sqlite, + expected_hash_length: 64, + }, + CredentialTestCase { + description: "PostgreSQL credential lifecycle and atomic audit", + backend: Backend::Postgres, + expected_hash_length: 64, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: Some("127.0.0.1".parse().expect("loopback")), + }; + let fault = TransactionFault { + description: "security audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!( + create_principal( + store, + &context, + ("orders-ci", PrincipalKind::ServiceAccount) + ) + .await + .is_err() + ); + assert!(create_location(store, "orders", &context).await.is_err()); + assert_eq!(location_state(&replica).await, (0, 0)); + remove_fault(store, &fault).await; + let principal_id = create_principal( + store, + &context, + ("orders-ci", PrincipalKind::ServiceAccount), + ) + .await + .expect("principal creation and audit rolled back together"); + let location_id = create_location(store, "orders", &context) + .await + .expect("location"); + assert_eq!(location_state(&replica).await, (1, 1)); + let scopes = [ + CredentialScope::Principal { principal_id }, + CredentialScope::Location { location_id }, + ]; + let expiry = Utc::now() + Duration::hours(1); + for scope in scopes { + let issued = issue_token(store, &context, scope, expiry) + .await + .expect("issue scoped token"); + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("replica verifies"), + Some(issued.identity.clone()) + ); + assert!( + !format!("{issued:?}").contains(&issued.bearer), + "debug output redacts secrets" + ); + let replacement = ["0", "1"][usize::from(issued.bearer.ends_with('0'))]; + let tampered = + format!("{}{replacement}", &issued.bearer[..issued.bearer.len() - 1]); + for invalid in [ + &tampered, + "", + "not-a-token", + &format!("{}.{}", Uuid::now_v7(), "0".repeat(64)), + ] { + assert!( + authenticate_token(&replica, invalid) + .await + .expect("invalid credential") + .is_none() + ); + } + install_fault(store, &fault).await; + assert!( + revoke_token(store, &context, issued.identity.token_id) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("failed revocation rolled back") + .is_some() + ); + assert!( + revoke_token(store, &context, issued.identity.token_id) + .await + .expect("revoke") + ); + assert!( + !revoke_token(&replica, &context, issued.identity.token_id) + .await + .expect("retry revoke") + ); + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("revoked token") + .is_none() + ); + assert!( + issue_token(store, &context, scope, Utc::now() - Duration::seconds(1)) + .await + .is_err() + ); + let stored = stored_hashes(store).await; + assert!( + stored + .iter() + .all(|hash| hash.len() == test_case.expected_hash_length + && hash != &issued.bearer), + "{}", + test_case.description + ); + } + for scope in [ + CredentialScope::Principal { + principal_id: Uuid::now_v7(), + }, + CredentialScope::Location { + location_id: Uuid::now_v7(), + }, + CredentialScope::Run { + run_id: Uuid::now_v7(), + claim_id: Uuid::now_v7(), + }, + ] { + assert!( + issue_token(store, &context, scope, expiry).await.is_err(), + "inactive scopes cannot acquire credentials" + ); + } + let published = publish_snapshot(store, &snapshot(location_id)) + .await + .expect("snapshot"); + queue( + store, + &request((location_id, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let scope = CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let issued = issue_token(store, &context, scope, expiry) + .await + .expect("mint claimed-run token"); + assert_eq!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("worker token"), + Some(issued.identity.clone()) + ); + assert!( + issue_token( + store, + &context, + CredentialScope::Run { + run_id: claim.request.run_id, + claim_id: Uuid::now_v7() + }, + expiry + ) + .await + .is_err() + ); + finish(store, &lease, &claim).await; + assert!( + authenticate_token(&replica, &issued.bearer) + .await + .expect("terminal token expires immediately") + .is_none() + ); + assert!( + issue_token(store, &context, scope, expiry).await.is_err(), + "a terminal claim cannot mint a new token" + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs new file mode 100644 index 0000000000..9e6edc1c94 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_contracts.rs @@ -0,0 +1,10 @@ +//! Credential contracts compiled within the credential owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "locations/helpers.rs"] +pub(crate) mod location_helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_types.rs b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs new file mode 100644 index 0000000000..c2efe4bf2a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/credentials/test_types.rs @@ -0,0 +1,8 @@ +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct CredentialTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_hash_length: usize, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/helpers.rs b/crates/sqlbuild-orchestration/tests/platform/helpers.rs new file mode 100644 index 0000000000..e2da7ca896 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/helpers.rs @@ -0,0 +1,211 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use axum::Json; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; + +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::publish_run_recall::publish_run_recall; +use crate::runs::models::{RecallBatch, RecallRecord}; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::models::ControlStore; +use crate::store::tests::helpers::read_recall_events; +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use sqlbuild_recall::records::models::RecallValue; +use sqlbuild_recall::schema::models::Table; +use tokio::task::JoinHandle; +use uuid::Uuid; + +use crate::platform::tests::test_types::{Observation, WebhookReceiver}; +use crate::runs::tests::helpers::{queue, request}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + let future = Utc::now() + TimeDelta::days(10); + let mut triggers = BTreeMap::new(); + for index in 0..101 { + triggers.insert( + format!("daily_orders_{index}"), + json!({ + "description": "Refresh orders", "job_key": "orders", + "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", future], + "refresh_at": "2020-01-01T00:00:00Z"} + }), + ); + } + for (name, status) in [("expired_orders", "running"), ("paused_orders", "paused")] { + triggers.insert(name.into(), json!({ + "description": "Refresh orders", "job_key": "orders", "default_status": status, + "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z"], "refresh_at": "2020-01-01T00:00:00Z"} + })); + } + triggers.insert( + "future_orders".into(), + json!({ + "description": "Future orders refresh", "job_key": "orders", + "schedule": {"kind": "calendar", "times": [future], "refresh_at": future} + }), + ); + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: None, + environment_hash: "orders-watchdog-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([ + ( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ), + ("triggers".into(), json!(triggers)), + ]), + } +} + +pub(crate) async fn wait_for_scans( + store: &ControlStore, + replicas: &BTreeSet, + tasks: &[JoinHandle>], +) -> Vec { + tokio::time::timeout(Duration::from_secs(10), async { + let mut cursor = 0; + let mut observations = Vec::new(); + let mut scanned = BTreeSet::new(); + for _ in 0..1000 { + let Some(_) = replicas.difference(&scanned).next() else { + return observations; + }; + assert!( + tasks.iter().all(|task| !task.is_finished()), + "watchdog failed before scanning" + ); + let batch: Vec = read_recall_events(store, cursor, 1000) + .await + .expect("read health facts") + .into_iter() + .map(|row| serde_json::from_value(json!(row)).expect("typed platform event")) + .collect(); + cursor = batch.last().map_or(cursor, |row| row.storage_id); + scanned.extend(batch.iter().filter_map(|row| { + let PlatformEvent::Heartbeat { replica_id } = row.payload.platform else { + return None; + }; + Some(replica_id) + })); + observations.extend(batch); + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("watchdog exhausted scan completion attempts"); + }) + .await + .expect("both replicas finish a paged scan without a scheduler") +} + +pub(crate) async fn receive_webhook( + State(receiver): State, + headers: HeaderMap, + Json(event): Json, +) -> impl IntoResponse { + assert_eq!( + headers.get("Idempotency-Key").expect("deduplication key"), + event.event_id.to_string().as_str() + ); + let mut state = receiver.state.lock().expect("receiver state"); + state.requests.push(event); + receiver.delivered.send_replace(state.requests.len()); + ( + StatusCode::from_u16(state.status).expect("configured response status"), + [(axum::http::header::LOCATION, "/events")], + ) +} + +pub(crate) async fn wait_for_webhook( + receiver: &WebhookReceiver, + dispatcher: &JoinHandle>, + count: usize, +) { + let mut delivered = receiver.delivered.subscribe(); + tokio::time::timeout( + Duration::from_secs(10), + delivered.wait_for(|received| *received >= count), + ) + .await + .expect("dispatcher drains more than one bounded page") + .expect("webhook receiver remains available"); + assert!( + !dispatcher.is_finished(), + "delivery loop stopped before draining" + ); +} + +pub(crate) async fn publish_platform_impostor(store: &ControlStore, identity: (Uuid, Uuid)) { + let mut requested = request(identity, 42, 0, &["orders"]); + requested.image_id = None; + queue(store, &requested).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("work"); + let batch = RecallBatch { + invocation_id: "platform/impostor".into(), + records: vec![RecallRecord { + table: Table::Events, + values: BTreeMap::from([ + ( + "event_id".into(), + RecallValue::Text(Uuid::now_v7().to_string()), + ), + ( + "event_type".into(), + RecallValue::Text("platform_heartbeat".into()), + ), + ("event_schema_version".into(), RecallValue::Integer(1)), + ("event_kind".into(), RecallValue::Text("runtime".into())), + ("severity".into(), RecallValue::Text("info".into())), + ( + "occurred_at".into(), + RecallValue::Text("2026-01-01T00:00:00Z".into()), + ), + ("producer".into(), RecallValue::Text("platform".into())), + ("invocation_sequence".into(), RecallValue::Integer(0)), + ( + "payload".into(), + RecallValue::Text( + json!({"platform": {"kind": "heartbeat", "replica_id": Uuid::now_v7()}}) + .to_string(), + ), + ), + ]), + }], + }; + let result = publish_run_recall(store, (claim.request.run_id, claim.claim_id), &batch) + .await + .expect("publish untrusted producer"); + assert_eq!(result.delivered, [(Table::Events, 1)]); + assert!(result.conflicts.is_empty()); +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs new file mode 100644 index 0000000000..0f5c53a679 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs @@ -0,0 +1,10 @@ +//! Platform contracts compiled within the platform owner. + +#[path = "helpers.rs"] +mod helpers; +#[path = "test_types.rs"] +mod test_types; +#[path = "test_watchdog.rs"] +mod test_watchdog; +#[path = "test_webhook.rs"] +mod test_webhook; diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs new file mode 100644 index 0000000000..52ecb92e0c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -0,0 +1,53 @@ +use std::sync::{Arc, Mutex}; + +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; +use chrono::{DateTime, Utc}; +use serde::Deserialize; + +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct WatchdogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) leaders: usize, + pub(crate) expected_no_leader: bool, + pub(crate) expected_overdue: usize, + pub(crate) expected_low_horizons: usize, +} + +pub(crate) struct LocationWatchdogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_broken: usize, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct Observation { + pub(crate) storage_id: i64, + pub(crate) occurred_at: DateTime, + pub(crate) payload: PlatformPayload, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct PlatformPayload { + pub(crate) platform: PlatformEvent, +} + +pub(crate) struct PlatformSinkTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) initial_status: u16, + pub(crate) expected_events: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct WebhookReceiver { + pub(crate) state: Arc>, + pub(crate) delivered: tokio::sync::watch::Sender, +} + +#[derive(Debug)] +pub(crate) struct WebhookState { + pub(crate) status: u16, + pub(crate) requests: Vec, +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs new file mode 100644 index 0000000000..fa3574df89 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_watchdog.rs @@ -0,0 +1,343 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::main::run_watchdog::run_watchdog; +use crate::platform::models::{LocationHealth, LocationHealthReport, PlatformEvent}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{Parameter, TransactionMode}; +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::tests::helpers::{snapshot, wait_for_scans}; +use crate::platform::tests::test_types::{LocationWatchdogTestCase, WatchdogTestCase}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_two_server_replicas_when_scheduling_is_stalled_then_health_facts_are_durable_deduplicated_and_independent_of_leadership() + { + let test_cases = [ + WatchdogTestCase { + description: "SQLite without a leader", + backend: Backend::Sqlite, + leaders: 0, + expected_no_leader: true, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "PostgreSQL without a leader", + backend: Backend::Postgres, + leaders: 0, + expected_no_leader: true, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "SQLite with a frozen leader", + backend: Backend::Sqlite, + leaders: 1, + expected_no_leader: false, + expected_overdue: 102, + expected_low_horizons: 1, + }, + WatchdogTestCase { + description: "PostgreSQL with a frozen leader", + backend: Backend::Postgres, + leaders: 1, + expected_no_leader: false, + expected_overdue: 102, + expected_low_horizons: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate health store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input + .documents + .get_mut("triggers") + .expect("trigger document")["removed_orders"] = + input.documents["triggers"]["expired_orders"].clone(); + publish_snapshot(store, &input) + .await + .expect("old definition"); + input.generation = 2; + input + .documents + .get_mut("triggers") + .expect("trigger document") + .as_object_mut() + .expect("trigger map") + .remove("removed_orders"); + publish_snapshot(store, &input) + .await + .expect("current definition"); + for _ in 0..test_case.leaders { + acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("leader"); + } + let second = second_store(store).await; + let replicas = BTreeSet::from([Uuid::now_v7(), Uuid::now_v7()]); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let tasks: Vec<_> = [store.clone(), second.clone()] + .into_iter() + .zip(replicas.iter()) + .map(|(store, id)| { + let id = *id; + let shutdown = shutdown.clone(); + tokio::spawn(async move { run_watchdog(&store, id, shutdown).await }) + }) + .collect(); + let observations = wait_for_scans(store, &replicas, &tasks).await; + let overdue: BTreeSet<_> = observations + .iter() + .filter_map(|row| { + let PlatformEvent::TickOverdue { trigger_id, .. } = row.payload.platform else { + return None; + }; + Some(trigger_id) + }) + .collect(); + assert_eq!( + overdue.len(), + test_case.expected_overdue, + "{}", + test_case.description + ); + let horizons: BTreeSet<_> = observations + .iter() + .filter_map(|row| { + let PlatformEvent::CalendarHorizonLow { trigger_id, .. } = row.payload.platform + else { + return None; + }; + Some(trigger_id) + }) + .collect(); + assert_eq!( + horizons.len(), + test_case.expected_low_horizons, + "paused and removed calendars are excluded; refresh cadence alone is not a low horizon" + ); + assert_eq!( + observations + .iter() + .any(|row| matches!(row.payload.platform, PlatformEvent::SchedulerNoLeader)), + test_case.expected_no_leader + ); + let logical_keys: BTreeSet<_> = observations + .iter() + .map(|row| { + ( + row.occurred_at, + serde_json::to_string(&row.payload.platform).expect("event identity"), + ) + }) + .collect(); + assert_eq!( + logical_keys.len(), + observations.len(), + "replicas do not duplicate an observation within its window" + ); + assert!( + observations + .iter() + .all(|row| row.occurred_at.timestamp().rem_euclid(30) == 0) + ); + stop.send(true).expect("shutdown both replicas"); + for task in tasks { + tokio::time::timeout(Duration::from_secs(2), task) + .await + .expect("prompt shutdown") + .expect("join watchdog") + .expect("watchdog stopped"); + } + close_store(second).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_stale_location_sessions_when_watchdogs_scan_then_latest_target_liveness_controls_paged_outage_events() + { + let test_cases = [ + LocationWatchdogTestCase { + description: "SQLite latest target receipts", + backend: Backend::Sqlite, + expected_broken: 104, + }, + LocationWatchdogTestCase { + description: "PostgreSQL latest target receipts", + backend: Backend::Postgres, + expected_broken: 104, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let archived = create_location(store, "inventory", "test-admin") + .await + .expect("archived location"); + create_location(store, "products", "test-admin") + .await + .expect("never observed location"); + for (location_id, count) in [(location, 105), (archived, 1)] { + for index in 0..count { + for _ in 0..2 { + report_location_health( + store, + location_id, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: Uuid::now_v7(), + target: format!("orders_{index:03}"), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register old session"); + } + } + } + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age sessions"); + let stale_at = transaction.now - TimeDelta::seconds(105); + let (transaction, _) = query( + transaction, + "UPDATE sqb_location_instances SET last_seen_at=$1", + vec![Parameter::Time(stale_at)], + ) + .await + .expect("age all old sessions"); + let now = transaction.now; + let (transaction, _) = query( + transaction, + "UPDATE sqb_locations SET archived_at=$1 WHERE location_id=$2", + vec![Parameter::Time(now), Parameter::Id(archived)], + ) + .await + .expect("archive location"); + commit(transaction).await.expect("commit fixture"); + // A fresh successor protects only its target, not another target of the same location. + let healthy_session = Uuid::now_v7(); + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: healthy_session, + target: "orders_104".into(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("healthy successor"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("age successor"); + let recent_at = transaction.now - TimeDelta::seconds(75); + let (transaction, _) = query( + transaction, + "UPDATE sqb_location_instances SET last_seen_at=$1 WHERE instance_id=$2", + vec![Parameter::Time(recent_at), Parameter::Id(healthy_session)], + ) + .await + .expect("two missed heartbeats"); + commit(transaction).await.expect("commit successor time"); + let second = second_store(store).await; + let replicas = BTreeSet::from([Uuid::now_v7(), Uuid::now_v7()]); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let tasks: Vec<_> = [store.clone(), second.clone()] + .into_iter() + .zip(replicas.iter()) + .map(|(store, id)| { + let id = *id; + let shutdown = shutdown.clone(); + tokio::spawn(async move { run_watchdog(&store, id, shutdown).await }) + }) + .collect(); + let observations = wait_for_scans(store, &replicas, &tasks).await; + let broken: BTreeSet<_> = observations + .iter() + .filter_map(|row| { + let PlatformEvent::LocationBroken { + location_id, + ref target, + last_seen_at, + } = row.payload.platform + else { + return None; + }; + assert_eq!( + location_id, location, + "archived and never-observed locations are excluded" + ); + assert_eq!( + last_seen_at, stale_at, + "payload retains the latest server receipt time" + ); + Some(target.clone()) + }) + .collect(); + assert_eq!( + broken, + (0..test_case.expected_broken) + .map(|index| format!("orders_{index:03}")) + .collect(), + "{}: both pages are read, old replicas deduplicate, and the healthy target is excluded", + test_case.description + ); + let keys: BTreeSet<_> = observations + .iter() + .map(|row| { + ( + row.occurred_at, + serde_json::to_string(&row.payload.platform).expect("identity"), + ) + }) + .collect(); + assert_eq!( + keys.len(), + observations.len(), + "replicas share observation keys" + ); + stop.send(true).expect("stop replicas"); + for task in tasks { + tokio::time::timeout(Duration::from_secs(2), task) + .await + .expect("prompt shutdown") + .expect("join") + .expect("watchdog stopped"); + } + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs new file mode 100644 index 0000000000..d2aeeea1c8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_webhook.rs @@ -0,0 +1,193 @@ +use std::collections::BTreeSet; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::Router; +use axum::routing::post; +use uuid::Uuid; + +use crate::platform::_helpers::webhook_delivery::deliver_platform_events; +use crate::platform::main::connect_platform_webhook::connect_platform_webhook; +use crate::platform::main::run_platform_sink::run_platform_sink; +use crate::platform::main::run_watchdog::run_watchdog; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; + +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::tests::helpers::{ + publish_platform_impostor, receive_webhook, snapshot, wait_for_scans, wait_for_webhook, +}; +use crate::platform::tests::test_types::{PlatformSinkTestCase, WebhookReceiver, WebhookState}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_location_outage_when_webhook_delivery_retries_then_all_health_facts_survive_restart_and_lost_acknowledgements() + { + let test_cases = [ + PlatformSinkTestCase { + description: "SQLite durable platform delivery", + backend: Backend::Sqlite, + initial_status: 503, + expected_events: 105, + }, + PlatformSinkTestCase { + description: "PostgreSQL durable platform delivery", + backend: Backend::Postgres, + initial_status: 307, + expected_events: 105, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate delivery store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish schedules"); + // No location process and no scheduler are running. + let replica_id = Uuid::now_v7(); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let watchdog_store = store.clone(); + let tasks = [tokio::spawn(async move { + run_watchdog(&watchdog_store, replica_id, shutdown).await + })]; + let observations = wait_for_scans(store, &BTreeSet::from([replica_id]), &tasks).await; + stop.send(true).expect("stop watchdog"); + for task in tasks { + task.await + .expect("watchdog task") + .expect("watchdog stopped"); + } + assert_eq!( + observations.len(), + test_case.expected_events, + "{}", + test_case.description + ); + publish_platform_impostor(store, (location, published.snapshot_id)).await; + + let state = Arc::new(Mutex::new(WebhookState { + status: test_case.initial_status, + requests: Vec::new(), + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("webhook listener"); + let endpoint = format!( + "http://delivery:sample-password@{}/events?token=sample-token", + listener.local_addr().expect("address") + ); + let receiver = WebhookReceiver { + state: state.clone(), + delivered: tokio::sync::watch::channel(0).0, + }; + let app = Router::new() + .route("/events", post(receive_webhook)) + .with_state(receiver.clone()); + let (http_stop, http_shutdown) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + drop(http_shutdown.await); + }) + .await + }); + let sink = connect_platform_webhook(&endpoint).expect("configure webhook"); + let diagnostic = format!("{sink:?}"); + assert!(!diagnostic.contains("sample-password")); + assert!(!diagnostic.contains("sample-token")); + assert!(deliver_platform_events(store, &sink).await.is_err()); + assert_eq!( + state.lock().expect("receiver").requests.len(), + 1, + "failure stops the page" + ); + state.lock().expect("receiver").status = 204; + let fault = TransactionFault { + description: "lose a successful webhook acknowledgement", + table: "sqb_platform_sink_deliveries", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'test receipt failure')", + postgres_body: "RAISE EXCEPTION 'test receipt failure'", + }; + install_fault(store, &fault).await; + assert!(deliver_platform_events(store, &sink).await.is_err()); + remove_fault(store, &fault).await; + let second = second_store(store).await; + let restarted = connect_platform_webhook(&endpoint).expect("restart webhook client"); + let (sink_stop, sink_shutdown) = tokio::sync::watch::channel(false); + let sink_store = second.clone(); + let dispatcher = tokio::spawn(async move { + run_platform_sink(&sink_store, restarted, sink_shutdown).await + }); + wait_for_webhook(&receiver, &dispatcher, test_case.expected_events + 2).await; + sink_stop.send(true).expect("stop dispatcher"); + tokio::time::timeout(Duration::from_secs(2), dispatcher) + .await + .expect("bounded dispatcher shutdown") + .expect("dispatcher task") + .expect("dispatcher stopped"); + // Shutdown may cancel the last receipt after the remote endpoint accepted it. + assert!( + deliver_platform_events(store, &sink) + .await + .expect("reconcile in-flight delivery") + <= 1 + ); + assert_eq!( + deliver_platform_events(store, &sink) + .await + .expect("durable cursor"), + 0 + ); + { + let receiver = state.lock().expect("receiver"); + let requests = &receiver.requests; + assert!( + (test_case.expected_events + 2..=test_case.expected_events + 3) + .contains(&requests.len()) + ); + assert_eq!(requests[0], requests[1], "HTTP retry retains full identity"); + assert_eq!( + requests[1], requests[2], + "lost receipt retries the same fact" + ); + assert_eq!( + requests[2..] + .iter() + .map(|event| event.event_id) + .collect::>() + .len(), + test_case.expected_events + ); + let mut delivered = requests[2..].to_vec(); + delivered.dedup(); + assert_eq!(delivered.len(), observations.len()); + for (event, observation) in delivered.iter().zip(&observations) { + assert_eq!(event.storage_id, observation.storage_id); + assert_eq!(event.occurred_at, observation.occurred_at); + assert_eq!(event.payload.platform, observation.payload.platform); + } + } + http_stop.send(()).expect("stop receiver"); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .expect("bounded shutdown") + .expect("server task") + .expect("server stopped"); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs new file mode 100644 index 0000000000..6b2c0e2b1d --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -0,0 +1,72 @@ +//! Real slot accounting and successful completion through the public worker lifecycle. + +use std::collections::BTreeMap; + +use crate::runs::main::change_run::change_run; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome, +}; +use crate::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { + let sql = "SELECT group_key, count(*) FROM sqb_concurrency_slots GROUP BY group_key"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect SQLite") + .prepare(sql) + .expect("slot query") + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("SQLite slot counts") + .collect::>() + .expect("decode slot counts"); + }; + sqlx_core::query_as::query_as::<_, (String, i64)>(sql) + .fetch_all(pool) + .await + .expect("PostgreSQL slot counts") + .into_iter() + .collect() +} + +pub(crate) async fn succeed(store: &ControlStore, claim: &RunClaim) { + let mut changes = vec![RunChange::Started]; + changes.extend( + claim + .request + .selection + .iter() + .map(|node| RunChange::NodeFinished { + node_key: node.clone(), + outcome: NodeOutcome::Success, + rows_affected: Some(1), + error_code: None, + skipped_because_of: None, + }), + ); + changes.push(RunChange::Finished { + outcome: RunOutcome::Success, + reason: "synthetic child completed".into(), + exit_code: Some(0), + }); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("worker fact"); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs new file mode 100644 index 0000000000..12663d3921 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -0,0 +1,106 @@ +//! Run fixtures exercise native queues and Recall against isolated real databases. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::runs::_helpers::requests::publication::publish_request; +use crate::runs::models::{RunKind, RunOrigin, RunRequest}; +use crate::scheduler::models::Overlap; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlStore, TransactionMode}; +use serde_json::json; +use uuid::Uuid; + +pub(crate) async fn request_run( + store: &ControlStore, + request: &RunRequest, +) -> Result { + let transaction = begin(store, TransactionMode::Write).await?; + let (transaction, applied) = publish_request(transaction, request).await?; + commit(transaction).await?; + Ok(applied) +} + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: Some("sha256:synthetic-orders-image".into()), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: ["customers", "inventory", "orders", "products"] + .into_iter() + .map(|name| { + ( + name.into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }, + ) + }) + .collect(), + documents: BTreeMap::new(), + } +} + +pub(crate) fn request( + (location_id, snapshot_id): (Uuid, Uuid), + id: u128, + priority: i64, + nodes: &[&str], +) -> RunRequest { + let selection: BTreeSet = nodes.iter().map(|name| (*name).into()).collect(); + let run_id = Uuid::from_u128(id); + RunRequest { + run_id, + location_id, + target: "development".into(), + snapshot_id, + image_id: Some("sha256:synthetic-orders-image".into()), + kind: RunKind::Adhoc, + origin: RunOrigin::Manual, + job_key: None, + parent_run_id: None, + root_run_id: run_id, + trigger_id: None, + tick_id: None, + rerun_of_run_id: None, + requested_by: "test-operator".into(), + priority, + write_set: selection.clone(), + selection, + settings: BTreeMap::new(), + window: None, + concurrency_groups: BTreeSet::new(), + overlap: Overlap::Allow, + } +} + +pub(crate) async fn queue(store: &ControlStore, request: &RunRequest) { + assert!( + request_run(store, request) + .await + .expect("queue pinned request") + ); + assert!( + !request_run(store, request) + .await + .expect("retry pinned request") + ); + let mut changed = request.clone(); + changed.priority += 1; + assert!( + request_run(store, &changed).await.is_err(), + "a reused run ID cannot change its request" + ); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs new file mode 100644 index 0000000000..c69347e480 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/monitor/helpers.rs @@ -0,0 +1,68 @@ +//! Advance persisted liveness, not the scheduler clock, to exercise real monitoring transactions. + +use std::collections::BTreeMap; + +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunRequest}; +use crate::scheduler::models::SchedulerLease; +use crate::store::models::ControlStore; +use chrono::{TimeDelta, Utc}; +use uuid::Uuid; + +use crate::runs::tests::helpers::queue; + +pub(crate) async fn start_run( + store: &ControlStore, + lease: &SchedulerLease, + input: &RunRequest, +) -> RunClaim { + queue(store, input).await; + let claim = claim_next_run(store, lease, &BTreeMap::new(), None) + .await + .expect("claim queued run") + .expect("run available"); + assert_eq!(claim.request.run_id, input.run_id); + change_run( + store, + RunAuthority::Worker { + run_id: input.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: input.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }, + ) + .await + .expect("start claimed worker"); + claim +} + +pub(crate) async fn age_heartbeat(store: &ControlStore, run_id: Uuid, seconds: i64) { + let timestamp = Utc::now() - TimeDelta::seconds(seconds); + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("open liveness fixture") + .execute( + "UPDATE sqb_run_liveness SET last_heartbeat_at=?1 WHERE run_id=?2", + rusqlite::params![ + timestamp.to_rfc3339_opts(chrono::SecondsFormat::Micros, true), + run_id.to_string() + ], + ) + .expect("age SQLite heartbeat"); + return; + }; + sqlx_core::query::query("UPDATE sqb_run_liveness SET last_heartbeat_at=$1 WHERE run_id=$2") + .bind(timestamp) + .bind(run_id) + .execute(pool) + .await + .expect("age PostgreSQL heartbeat"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs new file mode 100644 index 0000000000..5b31b83c2f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/termination/helpers.rs @@ -0,0 +1,30 @@ +//! Terminal facts exercise claim release through the public run lifecycle. + +use crate::runs::main::change_run::change_run; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunClaim, RunOutcome}; +use crate::scheduler::models::SchedulerLease; +use crate::store::models::ControlStore; +use uuid::Uuid; + +pub(crate) async fn finish(store: &ControlStore, lease: &SchedulerLease, claim: &RunClaim) { + let request = RunChangeRequest { + event_id: Uuid::new_v5(&claim.claim_id, b"test-finished"), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "synthetic worker exit".into(), + exit_code: Some(1), + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("terminate claimed work") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &request) + .await + .expect("retry terminal fact") + ); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs new file mode 100644 index 0000000000..d8c4151143 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -0,0 +1,532 @@ +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_run_claims::list_run_claims; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::RunStatus; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::succeed; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::terminal_helpers::finish; +use crate::runs::tests::test_types::{ + BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase, ScopedClaimTestCase, +}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_higher_priority_work_for_other_scopes_when_a_local_launcher_claims_then_only_its_location_and_target_are_reserved() + { + let test_cases = [ + ScopedClaimTestCase { + description: "SQLite local dispatch cannot reserve another location or target", + backend: Backend::Sqlite, + expected_selected_run: 1, + expected_unscoped_run: 3, + }, + ScopedClaimTestCase { + description: "PostgreSQL local dispatch cannot reserve another location or target", + backend: Backend::Postgres, + expected_selected_run: 1, + expected_unscoped_run: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "inventory", "test-admin") + .await + .expect("other location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let other_published = publish_snapshot(store, &snapshot(other)) + .await + .expect("other location snapshot"); + let mut secondary = snapshot(location); + secondary.target = "secondary".into(); + let secondary = publish_snapshot(store, &secondary) + .await + .expect("other target snapshot"); + let wanted = request((location, published.snapshot_id), 1, 0, &["orders"]); + let mut other_target = request((location, secondary.snapshot_id), 2, 10, &["orders"]); + other_target.target = "secondary".into(); + let other_location = request((other, other_published.snapshot_id), 3, 20, &["orders"]); + for input in [&wanted, &other_target, &other_location] { + queue(store, input).await; + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("scoped claim") + .expect("matching work"); + assert_eq!( + claim.request.run_id, + Uuid::from_u128(test_case.expected_selected_run), + "{}", + test_case.description + ); + assert_eq!( + list_run_claims(store, (location, "development"), None) + .await + .expect("recover queued claim"), + vec![claim.clone()], + "recovery includes a committed claim before launch" + ); + assert!( + list_run_claims(store, (location, "development"), Some(claim.request.run_id)) + .await + .expect("page after claimed run") + .is_empty() + ); + assert!( + list_run_claims(store, (other, "development"), None) + .await + .expect("unclaimed neighboring location") + .is_empty() + ); + assert!( + list_run_claims(store, (location, "secondary"), None) + .await + .expect("unclaimed neighboring target") + .is_empty() + ); + assert!( + claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")) + ) + .await + .expect("scope exhausted") + .is_none() + ); + finish(store, &lease, &claim).await; + assert!( + list_run_claims(store, (location, "development"), None) + .await + .expect("terminal claims are not recoverable") + .is_empty() + ); + let next = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("unrestricted claim") + .expect("other work was not reserved"); + assert_eq!( + next.request.run_id, + Uuid::from_u128(test_case.expected_unscoped_run), + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_overlapping_priority_queues_when_two_launchers_claim_then_conflicts_rollback_and_terminal_facts_release_claims() + { + let test_cases = [ + PriorityClaimsTestCase { + description: "SQLite claims prioritize independent write sets and release atomically", + backend: Backend::Sqlite, + expected_initial_claims: [2, 3], + expected_claim_events: 4, + expected_terminal_events: 2, + }, + PriorityClaimsTestCase { + description: "PostgreSQL claims prioritize independent write sets and release atomically", + backend: Backend::Postgres, + expected_initial_claims: [2, 3], + expected_claim_events: 4, + expected_terminal_events: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish snapshot"); + let scope = (location, published.snapshot_id); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + for input in [ + request(scope, 1, 1, &["orders"]), + request(scope, 2, 3, &["orders"]), + request(scope, 3, 2, &["products"]), + ] { + queue(store, &input).await; + } + let second = second_store(store).await; + let limits = BTreeMap::new(); + let (first, other) = tokio::join!( + claim_next_run(store, &lease, &limits, None), + claim_next_run(&second, &lease, &limits, None) + ); + let first = first.expect("first claimant").expect("claim available"); + let other = other + .expect("second claimant") + .expect("independent claim available"); + let claimed: std::collections::BTreeSet<_> = + [first.request.run_id, other.request.run_id] + .into_iter() + .collect(); + assert_eq!( + claimed, + test_case + .expected_initial_claims + .into_iter() + .map(Uuid::from_u128) + .collect(), + "{}", + test_case.description + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("blocked run stays queued") + .is_none() + ); + queue(store, &request(scope, 4, 9, &["customers", "orders"])).await; + queue(store, &request(scope, 5, 0, &["customers"])).await; + let independent = claim_next_run(store, &lease, &limits, None) + .await + .expect("try partially conflicting candidate") + .expect("independent work proceeds"); + assert_eq!( + independent.request.run_id, + Uuid::from_u128(5), + "customers was not leaked by the failed customers/orders claim" + ); + let orders = [&first, &other] + .into_iter() + .find(|claim| claim.request.run_id == Uuid::from_u128(2)) + .expect("orders owner"); + finish(store, &lease, orders).await; + finish(store, &lease, &independent).await; + let next = claim_next_run(store, &lease, &limits, None) + .await + .expect("terminal cleanup committed") + .expect("next priority run claims both nodes"); + assert_eq!(next.request.run_id, Uuid::from_u128(4)); + let events = read_recall_events(store, 0, 100) + .await + .expect("read immutable run stream"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_claimed") + .count(), + test_case.expected_claim_events, + "{}", + test_case.description + ); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_terminated") + .count(), + test_case.expected_terminal_events, + "{}", + test_case.description + ); + let runs = list_runs(store, (location, "development"), 100) + .await + .expect("read atomic projections"); + assert_eq!( + runs.iter() + .filter(|run| run.status == RunStatus::Failed) + .count(), + 2 + ); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_named_group_limits_when_capacity_is_lowered_then_active_high_slots_still_count_and_terminal_cleanup_unblocks_work() + { + let test_cases = [ + GroupLimitTestCase { + description: "SQLite occupied high slots still count after lowering a group limit", + backend: Backend::Sqlite, + expected_unblocked_run: 3, + }, + GroupLimitTestCase { + description: "PostgreSQL occupied high slots still count after lowering a group limit", + backend: Backend::Postgres, + expected_unblocked_run: 3, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate run store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("publish snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut limits = BTreeMap::from([( + "warehouse".into(), + NonZeroU32::new(2).expect("nonzero limit"), + )]); + for id in 1..=3 { + let mut input = request((location, published.snapshot_id), id, 0, &["orders"]); + input.write_set.clear(); + input.concurrency_groups.insert("warehouse".into()); + queue(store, &input).await; + } + assert!( + claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .is_err(), + "unknown groups cannot silently become unlimited" + ); + let first = claim_next_run(store, &lease, &limits, None) + .await + .expect("first slot") + .expect("available"); + let second = claim_next_run(store, &lease, &limits, None) + .await + .expect("second slot") + .expect("available"); + assert_eq!(first.request.run_id, Uuid::from_u128(1)); + assert_eq!(second.request.run_id, Uuid::from_u128(2)); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("group full") + .is_none() + ); + finish(store, &lease, &first).await; + limits.insert( + "warehouse".into(), + NonZeroU32::new(1).expect("nonzero lowered limit"), + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("high slot counts after limit reduction") + .is_none() + ); + finish(store, &lease, &second).await; + let third = claim_next_run(store, &lease, &limits, None) + .await + .expect("released group") + .expect("third queued run"); + assert_eq!( + third.request.run_id, + Uuid::from_u128(test_case.expected_unblocked_run), + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_more_than_one_page_of_blocked_runs_when_claiming_then_lower_priority_independent_work_is_not_starved() + { + let test_cases = [ + BacklogClaimTestCase { + description: "SQLite scans past more than a full page of blocked work", + backend: Backend::Sqlite, + expected_independent_run: 105, + }, + BacklogClaimTestCase { + description: "PostgreSQL scans past more than a full page of blocked work", + backend: Backend::Postgres, + expected_independent_run: 105, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let mut lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + queue(store, &request(scope, 1, 0, &["orders"])).await; + claim_next_run(store, &lease, &limits, None) + .await + .expect("claim blocker") + .expect("available"); + for batch in (2..=104).collect::>().chunks(20) { + for id in batch { + queue(store, &request(scope, *id, 0, &["orders"])).await; + } + lease = crate::scheduler::main::renew_lease::renew_lease(store, &lease) + .await + .expect("keep leader live while preparing backlog"); + } + queue(store, &request(scope, 105, -1, &["customers"])).await; + let started = std::time::Instant::now(); + let independent = claim_next_run(store, &lease, &limits, None) + .await + .expect("scan past a full blocked page") + .expect("independent work"); + assert_eq!( + independent.request.run_id, + Uuid::from_u128(test_case.expected_independent_run), + "{}", + test_case.description + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "claim scan exceeded five seconds: {}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_blocked_multi_limit_waiter_when_younger_work_arrives_then_shared_limits_drain_without_blocking_unrelated_work() + { + let test_cases = [ + GroupLimitTestCase { + description: "SQLite protects priority across atomic limit sets", + backend: Backend::Sqlite, + expected_unblocked_run: 4, + }, + GroupLimitTestCase { + description: "PostgreSQL protects priority across atomic limit sets", + backend: Backend::Postgres, + expected_unblocked_run: 4, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + for (id, priority, nodes, groups) in [ + (1, 100, &["inventory"][..], &["inventory_limit"][..]), + ( + 2, + 10, + &["orders"][..], + &["orders_limit", "inventory_limit"][..], + ), + (3, 1, &["customers"][..], &["orders_limit"][..]), + (4, 0, &["products"][..], &[][..]), + ] { + let mut input = request(scope, id, priority, nodes); + input.concurrency_groups = groups.iter().map(|key| (*key).into()).collect(); + queue(store, &input).await; + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one")), + ("inventory_limit".into(), NonZeroU32::new(1).expect("one")), + ]); + let blocker = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("highest priority"); + assert_eq!(blocker.request.run_id, Uuid::from_u128(1)); + let unrelated = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("unrelated"); + assert_eq!( + unrelated.request.run_id, + Uuid::from_u128(test_case.expected_unblocked_run), + "{}", + test_case.description + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("wait") + .is_none() + ); + succeed(store, &blocker).await; + let protected = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("older waiter"); + assert_eq!(protected.request.run_id, Uuid::from_u128(2)); + succeed(store, &protected).await; + let younger = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("younger retry"); + assert_eq!(younger.request.run_id, Uuid::from_u128(3)); + succeed(store, &younger).await; + succeed(store, &unrelated).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs new file mode 100644 index 0000000000..b9eb5090c4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -0,0 +1,117 @@ +use std::collections::BTreeMap; + +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::models::TickDecision; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::CoalescedRunTestCase; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks_wait_in_a_new_run_instead_of_covering_executing_work() + { + let test_cases = [ + CoalescedRunTestCase { + description: "SQLite coalesces only work that has not been claimed", + backend: Backend::Sqlite, + expected_runs: 2, + }, + CoalescedRunTestCase { + description: "PostgreSQL coalesces only work that has not been claimed", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z"], "refresh_at": "2030-01-01T00:00:00Z"}}})); + let published = publish_snapshot(store, &input) + .await + .expect("first calendar publication"); + let trigger = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + evaluate_trigger(store, &lease, trigger) + .await + .expect("first tick"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim first run") + .expect("scheduled run"); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + input.generation = 2; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh calendar"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("second tick") + .expect("active trigger"); + assert_eq!(plan.actions.len(), 1); + let TickDecision::Requested { run_id: waiting } = plan.actions[0].decision else { + panic!("future work must not be covered by an already claimed run"); + }; + assert_ne!(waiting, claim.request.run_id); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!([ + "2020-01-01T00:00:00Z", + "2020-01-02T00:00:00Z", + "2020-01-03T00:00:00Z" + ]); + publish_snapshot(store, &input) + .await + .expect("next calendar refresh"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("third tick") + .expect("active trigger"); + assert_eq!( + plan.actions[0].decision, + TickDecision::Covered { + covered_by_run_id: waiting + } + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("one executing and one waiting") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs new file mode 100644 index 0000000000..530eb54c35 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -0,0 +1,28 @@ +#[path = "claims/helpers.rs"] +mod claim_helpers; +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "monitor/helpers.rs"] +mod monitor_helpers; +#[path = "termination/helpers.rs"] +pub(crate) mod terminal_helpers; +#[path = "test_claims.rs"] +mod test_claims; +#[path = "test_coalesce.rs"] +mod test_coalesce; +#[path = "test_faults.rs"] +mod test_faults; +#[path = "test_group_admission.rs"] +mod test_group_admission; +#[path = "test_groups.rs"] +mod test_groups; +#[path = "test_lifecycle.rs"] +mod test_lifecycle; +#[path = "test_monitor.rs"] +mod test_monitor; +#[path = "test_requests.rs"] +mod test_requests; +#[path = "test_reruns.rs"] +mod test_reruns; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_faults.rs b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs new file mode 100644 index 0000000000..f784d6485b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_faults.rs @@ -0,0 +1,177 @@ +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{ClaimFailureTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_failures_during_claim_publication_when_retried_then_claims_slots_liveness_facts_and_projection_commit_together() + { + let test_cases = [ + ClaimFailureTestCase { + description: "SQLite rolls back every claim-publication failure window", + backend: Backend::Sqlite, + expected_run: 1, + }, + ClaimFailureTestCase { + description: "PostgreSQL rolls back every claim-publication failure window", + backend: Backend::Postgres, + expected_run: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let mut input = request( + (location, published.snapshot_id), + 1, + 1, + &["customers", "orders"], + ); + input.concurrency_groups.insert("warehouse".into()); + queue(store, &input).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([( + "warehouse".into(), + NonZeroU32::new(1).expect("positive limit"), + )]); + let facts = read_recall_events(store, 0, 100) + .await + .expect("requested and queued facts"); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("queued projection"); + for (table, operation) in [ + ("sqb_write_claims", "INSERT"), + ("sqb_concurrency_slots", "INSERT"), + ("sqb_run_liveness", "INSERT"), + ("recall_events", "INSERT"), + ("sqb_runs", "UPDATE"), + ] { + let point = TransactionFault { + description: "claim publication failure", + table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected claim failure')", + postgres_body: "RAISE EXCEPTION 'injected claim failure'", + }; + install_fault(store, &point).await; + assert!( + claim_next_run(store, &lease, &limits, None).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back claim fact"), + facts + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("rolled back claim projection"), + projected + ); + } + let expiry = TransactionFault { + description: "claim fence expires before commit", + table: "sqb_run_liveness", + operation: "INSERT", + condition: "", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + }; + install_fault(store, &expiry).await; + assert!( + matches!( + claim_next_run(store, &lease, &limits, None).await, + Err(ControlError::Fenced) + ), + "{}", + expiry.description + ); + remove_fault(store, &expiry).await; + let claim = claim_next_run(store, &lease, &limits, None) + .await + .expect("retry complete claim") + .expect("no leaked claims or slots"); + assert_eq!(claim.request, input); + assert_eq!( + claim.request.run_id, + Uuid::from_u128(test_case.expected_run), + "{}", + test_case.description + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("already claimed run is not relaunched") + .is_none() + ); + expire_lease(store).await; + let next = acquire_lease(store, Uuid::now_v7()) + .await + .expect("take over") + .expect("new leader"); + let launched = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: input.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Launched { + container_id: Some("synthetic-orders-worker".into()), + }, + }; + assert!( + matches!( + change_run(store, RunAuthority::Scheduler(&lease), &launched).await, + Err(ControlError::Fenced) + ), + "a former leader cannot acknowledge a launch after takeover" + ); + assert!( + change_run(store, RunAuthority::Scheduler(&next), &launched) + .await + .expect("successor reconciles the existing claim") + ); + assert!( + !heartbeat_run(store, (input.run_id, claim.claim_id)) + .await + .expect("workers remain alive through scheduler failover") + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs new file mode 100644 index 0000000000..015e0ca74c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -0,0 +1,808 @@ +//! Real database admission preserves completion orders, capacities and fair retries across replicas. + +use std::collections::{BTreeMap, BTreeSet}; +use std::num::NonZeroU32; + +use crate::credentials::models::AuditContext; +use crate::runs::main::{ + advance_groups::advance_groups, claim_next_run::claim_next_run, list_runs::list_runs, + request_job_run::request_job_run, +}; +use crate::runs::models::{JobRunRequest, RunDispatch, RunStatus}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{ + AdmissionParent, GroupAdmissionTestCase, GroupQueueTestCase, MixedAdmissionTestCase, + RunLifecycleTestCase, +}; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queue_order_win() { + let test_cases = [ + MixedAdmissionTestCase { + description: "higher-priority physical request precedes parent admission", + physical_priority: 10, + parent_priority: 1, + expected_selections: ["inventory", "orders"], + }, + MixedAdmissionTestCase { + description: "earlier physical request wins at equal priority", + physical_priority: 1, + parent_priority: 1, + expected_selections: ["inventory", "orders"], + }, + MixedAdmissionTestCase { + description: "higher-priority parent precedes physical request", + physical_priority: 1, + parent_priority: 10, + expected_selections: ["orders", "inventory"], + }, + ]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"stage": {"description": "Build orders", "selection": ["orders"]}}), + ); + input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "concurrency_groups": ["warehouse"], "jobs": {"stage": []}}})); + let published = publish_snapshot(store, &input).await.expect("publish"); + let mut physical = request( + (location, published.snapshot_id), + 900, + test_case.physical_priority, + &["inventory"], + ); + physical.concurrency_groups.insert("warehouse".into()); + queue(store, &physical).await; + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: Uuid::from_u128(100), + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Exercise mixed queue ordering".into(), + priority: test_case.parent_priority, + window: None, + }, + ) + .await + .expect("queue group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = + BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one"))]); + let scope = (location, "development"); + for index in 0..test_case.expected_selections.len() { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect(test_case.description); + assert_eq!( + claim.request.selection, + BTreeSet::from([test_case.expected_selections[index].into()]), + "{} {backend:?}", + test_case.description + ); + assert_eq!(slot_counts(store).await.get("warehouse"), Some(&1)); + succeed(store, &claim).await; + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish parent"); + assert!( + list_runs(store, scope, 10) + .await + .expect("runs") + .iter() + .all(|run| run.status == RunStatus::Success) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknown_child_limits_do_not_block_it() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite dispatch excludes unserved prospective demand", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + RunLifecycleTestCase { + description: "PostgreSQL dispatch excludes unserved prospective demand", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let unserved = create_location(store, "inventory", "test-admin") + .await + .expect("unserved location"); + let served = create_location(store, "orders", "test-admin") + .await + .expect("served location"); + for location in [unserved, served] { + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({"stage": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": if location == unserved {vec!["inventory_limit"]} else {vec![]}}})); + input.documents.insert("groups".into(), json!({"daily": {"description": "Group orders", "selection": ["orders"], "jobs": {"stage": []}}})); + publish_snapshot(store, &input).await.expect("publish"); + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: location, + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Exercise launcher eligibility".into(), + priority: 1, + window: None, + }, + ) + .await + .expect("queue group"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let eligible = BTreeSet::from([(served, "development".into())]); + let dispatch = RunDispatch { + scope: Some((served, "development")), + eligible: Some(&eligible), + }; + advance_groups(store, &lease, (&limits, dispatch)) + .await + .expect("eligible parent advances"); + let claim = claim_next_run(store, &lease, &limits, dispatch) + .await + .expect("eligible dispatch") + .expect("eligible child"); + assert_eq!(claim.request.location_id, served); + succeed(store, &claim).await; + advance_groups(store, &lease, (&limits, dispatch)) + .await + .expect("finish eligible parent"); + assert!( + list_runs(store, (served, "development"), 10) + .await + .expect("served runs") + .iter() + .all(|run| run.status == test_case.expected_status), + "{}", + test_case.description + ); + assert!( + list_runs(store, (unserved, "development"), 10) + .await + .expect("unserved runs") + .iter() + .all(|run| run.status == RunStatus::Queued) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_then_every_group_finishes_within_limits() + { + let test_cases = [ + GroupAdmissionTestCase { + description: "capacity two permits both sides of a cycle", + limits: &[("orders_limit", 2), ("inventory_limit", 2)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[&[("orders_limit", 1), ("inventory_limit", 1)]], + }, + GroupAdmissionTestCase { + description: "asymmetric capacity still has a safe completion order", + limits: &[("orders_limit", 2), ("inventory_limit", 1)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "three-way cycle leaves one parent waiting", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["products_limit"]], + }, + AdmissionParent { + holds: &["products_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "a free first stage does not conceal a later cycle", + limits: &[("orders_limit", 1), ("inventory_limit", 1)], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&[], &["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&[], &["orders_limit"]], + }, + ], + expected_running: 1, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "a persistently blocked child lets its parent's peer drain shared limits", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit"], + children: &[&["inventory_limit", "products_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["products_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + GroupAdmissionTestCase { + description: "multiple parent limits do not hoist independent child limits", + limits: &[ + ("orders_limit", 1), + ("inventory_limit", 1), + ("products_limit", 1), + ("support_limit", 1), + ], + parents: &[ + AdmissionParent { + holds: &["orders_limit", "products_limit"], + children: &[&["inventory_limit"]], + }, + AdmissionParent { + holds: &["inventory_limit"], + children: &[&["orders_limit", "products_limit"]], + }, + AdmissionParent { + holds: &["support_limit"], + children: &[&["orders_limit"]], + }, + ], + expected_running: 2, + reductions: &[], + }, + ]; + for test_case in test_cases { + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let replica = second_store(store).await; + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + let mut jobs = serde_json::Map::new(); + let mut groups = serde_json::Map::new(); + let mut child_count = 0; + for (index, parent) in test_case.parents.iter().enumerate() { + let mut members = serde_json::Map::new(); + let mut selection = BTreeSet::new(); + for (stage, needs) in parent.children.iter().enumerate() { + let name = format!("orders_job_{index}_{stage}"); + let node = + ["orders", "customers", "inventory", "products"][child_count % 4]; + child_count += 1; + selection.insert(node); + jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); + let after: Vec = (0..stage) + .map(|earlier| format!("orders_job_{index}_{earlier}")) + .collect(); + members.insert(name, json!(after)); + } + groups.insert(format!("orders_group_{index}"), json!({"description": "Complete a synthetic group", "selection": selection, "concurrency_groups": parent.holds, "jobs": members})); + } + input.documents.insert("jobs".into(), json!(jobs)); + input.documents.insert("groups".into(), json!(groups)); + let published = publish_snapshot(store, &input) + .await + .expect("publish groups"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits: BTreeMap = test_case + .limits + .iter() + .map(|(key, count)| { + ( + (*key).into(), + NonZeroU32::new(*count).expect("positive capacity"), + ) + }) + .collect(); + let scope = (location, "development"); + for index in 0..test_case.parents.len() { + request_job_run( + store, + &actor, + &JobRunRequest { + // Higher-priority arrivals exercise safety, not only an older waiter's priority barrier. + run_id: Uuid::from_u128(1000 - index as u128), + location_id: location, + target: "development".into(), + job_key: format!("orders_group_{index}"), + reason: "Exercise capacity-aware admission".into(), + priority: index as i64 + 1, + window: None, + }, + ) + .await + .expect("request group"); + let (first, second) = tokio::join!( + advance_groups(store, &lease, (&limits, scope)), + advance_groups(&replica, &lease, (&limits, scope)) + ); + first.expect("first scheduler admits higher-priority arrival"); + second.expect("replica cannot admit an unsafe arrival"); + } + for _ in 0..test_case.parents.len() { + let (first, second) = tokio::join!( + advance_groups(store, &lease, (&limits, scope)), + advance_groups(&replica, &lease, (&limits, scope)) + ); + first.expect("first scheduler advance"); + second.expect("second scheduler advance"); + } + let initial = list_runs(store, scope, 100).await.expect("initial runs"); + assert_eq!( + initial + .iter() + .filter(|run| run.status == RunStatus::Running) + .count(), + test_case.expected_running, + "{} {backend:?}", + test_case.description + ); + let before = slot_counts(store).await; + for reduction in test_case.reductions { + let reduced = reduction + .iter() + .map(|(key, count)| { + ( + (*key).into(), + NonZeroU32::new(*count).expect("positive reduction"), + ) + }) + .collect(); + assert!( + claim_next_run(&replica, &lease, &reduced, Some(scope)) + .await + .expect("unsafe parent state must not fail physical dispatch") + .is_none() + ); + assert_eq!( + slot_counts(store).await, + before, + "blocked work cannot mutate claims" + ); + let unrelated = + request((location, published.snapshot_id), 777, 1, &["products"]); + queue(store, &unrelated).await; + let claim = claim_next_run(&replica, &lease, &reduced, Some(scope)) + .await + .expect("unrelated recovery progress") + .expect("unrelated work still dispatches"); + assert_eq!(claim.request.run_id, unrelated.run_id); + succeed(store, &claim).await; + } + for iteration in 0..child_count { + let owner = [store, &replica][iteration % 2]; + for _ in 0..test_case.parents.len() { + advance_groups(owner, &lease, (&limits, scope)) + .await + .expect("resume group from durable state"); + } + let claim = claim_next_run(owner, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect(test_case.description); + // Re-evaluate while a physical child still holds its temporary claims. + advance_groups(&replica, &lease, (&limits, scope)) + .await + .expect("advance during running claim"); + let occupied = slot_counts(store).await; + assert!( + occupied + .iter() + .all(|(key, count)| *count <= i64::from(limits[key].get())), + "{} {occupied:?}", + test_case.description + ); + succeed(owner, &claim).await; + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("aggregate final children"); + let finished = list_runs(&replica, scope, 100) + .await + .expect("finished groups"); + assert_eq!( + finished.len(), + test_case.parents.len() + child_count + test_case.reductions.len() + ); + assert!( + finished.iter().all(|run| run.status == RunStatus::Success), + "{} {finished:?}", + test_case.description + ); + assert!( + slot_counts(store).await.is_empty(), + "terminal parents release their own slots" + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } + } +} + +#[test] +fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_then_overlap_allow_still_runs() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite preserves overlap allow across generations", + backend: Backend::Sqlite, + expected_status: RunStatus::Running, + }, + RunLifecycleTestCase { + description: "PostgreSQL preserves overlap allow across generations", + backend: Backend::Postgres, + expected_status: RunStatus::Running, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"stage": {"description": "Build a pinned stage", "selection": ["orders"]}}), + ); + input.documents.insert("groups".into(), json!({"daily": {"description": "Allow independent generations", "selection": ["orders"], "overlap": "allow", "concurrency_groups": ["orders_limit"], "jobs": {"stage": []}}})); + let first = publish_snapshot(store, &input) + .await + .expect("first snapshot"); + let older = JobRunRequest { + run_id: Uuid::from_u128(2000), + location_id: location, + target: "development".into(), + job_key: "daily".into(), + reason: "Older group".into(), + priority: 1, + window: None, + }; + request_job_run(store, &actor, &older) + .await + .expect("older request"); + input.generation = 2; + input.documents.get_mut("jobs").expect("jobs")["stage"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily"]["concurrency_groups"] = + json!(["inventory_limit"]); + publish_snapshot(store, &input) + .await + .expect("second snapshot"); + let newer = JobRunRequest { + run_id: Uuid::from_u128(1000), + reason: "Independent replacement generation".into(), + ..older.clone() + }; + request_job_run(store, &actor, &newer) + .await + .expect("newer request"); + let mut physical = request((location, first.snapshot_id), 3000, 10, &["products"]); + physical.concurrency_groups.insert("orders_limit".into()); + queue(store, &physical).await; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one")), + ("inventory_limit".into(), NonZeroU32::new(1).expect("one")), + ]); + let scope = (location, "development"); + let blocker = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("physical claim") + .expect("higher priority blocker"); + assert_eq!(blocker.request.run_id, physical.run_id); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("independent group admission"); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == older.run_id) + .expect("older") + .status, + RunStatus::Queued + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == newer.run_id) + .expect("newer") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("new child") + .expect("allow generation can progress"); + assert_eq!(child.request.parent_run_id, Some(newer.run_id)); + assert_eq!(child.request.selection, ["inventory".to_owned()].into()); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish replacement parent"); + succeed(store, &blocker).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("old parent retries after blocker drains"); + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("old child") + .expect("older group makes progress"); + assert_eq!(child.request.parent_run_id, Some(older.run_id)); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("complete all groups"); + assert!( + list_runs(store, scope, 100) + .await + .expect("finished") + .iter() + .all(|run| run.status == RunStatus::Success) + ); + assert!(slot_counts(store).await.is_empty()); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelated_work_has_a_bounded_write_lock() + { + let test_cases = [ + GroupQueueTestCase { + description: "SQLite scans a full blocked group page without expanding unrelated bodies", + backend: Backend::Sqlite, + group_count: 105, + node_count: 3000, + expected_max_claim_millis: 5000, + }, + GroupQueueTestCase { + description: "PostgreSQL scans a full blocked group page without expanding unrelated bodies", + backend: Backend::Postgres, + group_count: 105, + node_count: 3000, + expected_max_claim_millis: 5000, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let actor = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + let template = input.nodes["orders"].clone(); + for index in 0..test_case.node_count { + let mut node = template.clone(); + node.definition["description"] = + json!(format!("Orders {index} {}", "details ".repeat(128))); + input.nodes.insert(format!("orders_{index}"), node); + } + let jobs = (0..test_case.group_count).map(|index| (format!("job_{index}"), json!({"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]}))).collect::>(); + let groups = (0..test_case.group_count).map(|index| (format!("group_{index}"), json!({"description": "Independent order group", "selection": ["orders"], "jobs": {format!("job_{index}"): []}}))).collect::>(); + input.documents.insert("jobs".into(), json!(jobs)); + input.documents.insert("groups".into(), json!(groups)); + input.documents.insert( + "columns".into(), + json!({"orders": "unused display metadata ".repeat(100000)}), + ); + let published = publish_snapshot(store, &input) + .await + .expect("publish large project"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one"))]); + let scope = (location, "development"); + let mut physical = request((location, published.snapshot_id), 1, 10, &["products"]); + physical.concurrency_groups.insert("warehouse".into()); + queue(store, &physical).await; + let blocker = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("blocker"); + for index in 0..test_case.group_count { + request_job_run( + store, + &actor, + &JobRunRequest { + run_id: Uuid::from_u128(1000 + index as u128), + location_id: location, + target: "development".into(), + job_key: format!("group_{index}"), + reason: "Exercise blocked group queue".into(), + priority: 1, + window: None, + }, + ) + .await + .expect("request group"); + } + let started = std::time::Instant::now(); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit independent parents"); + eprintln!( + "{:?}: {} parent admissions in {:?}", + test_case.backend, + test_case.group_count, + started.elapsed() + ); + let unrelated = request((location, published.snapshot_id), 2, -1, &["customers"]); + queue(store, &unrelated).await; + let started = std::time::Instant::now(); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("scan blocked children") + .expect("unrelated work"); + let elapsed = started.elapsed(); + eprintln!( + "{:?}: {} groups / {} nodes, claim transaction {:?}", + test_case.backend, test_case.group_count, test_case.node_count, elapsed + ); + assert_eq!( + claim.request.run_id, unrelated.run_id, + "{}", + test_case.description + ); + assert!( + elapsed.as_millis() < test_case.expected_max_claim_millis, + "{}: {elapsed:?}", + test_case.description + ); + succeed(store, &claim).await; + succeed(store, &blocker).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs new file mode 100644 index 0000000000..10f1b53bf8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -0,0 +1,424 @@ +//! Pinned groups coordinate real transactions, inherited slots, cancellation and replay. + +use std::collections::BTreeMap; +use std::num::NonZeroU32; + +use crate::credentials::models::AuditContext; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + list_runs::list_runs, request_job_run::request_job_run, run_execution::run_execution, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_and_parent_waits_for_terminal_evidence() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite group cancellation and replay", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + RunLifecycleTestCase { + description: "PostgreSQL group cancellation and replay", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.nodes.get_mut("orders").expect("orders").definition["deps"] = + json!(["customers"]); + input.documents.insert("jobs".into(), json!({ + "staging": {"description": "Stage customers", "selection": ["customers"], "concurrency_groups": ["warehouse"]}, + "marts": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["warehouse"]} + })); + input.documents.insert("groups".into(), json!({"daily_orders": { + "description": "Stage then build orders", "selection": ["customers", "orders"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["staging"]} + }})); + let published = publish_snapshot(store, &input) + .await + .expect("publish group"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build the group".into(), + priority: 1, + window: None, + }; + assert!( + request_job_run(store, &context, &request) + .await + .expect("request group") + ); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry group") + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = + BTreeMap::from([("warehouse".into(), NonZeroU32::new(1).expect("one slot"))]); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start virtual parent"); + let first = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim staging") + .expect("staging ready"); + assert_eq!(first.request.job_key.as_deref(), Some("staging")); + assert_eq!(first.request.parent_run_id, Some(request.run_id)); + assert_eq!(first.request.write_set, ["customers".to_owned()].into()); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("dependency waits") + .is_none() + ); + let authority = RunAuthority::Worker { + run_id: first.request.run_id, + claim_id: first.claim_id, + }; + for change in [ + RunChange::Started, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: NodeOutcome::Success, + rows_affected: Some(11), + error_code: None, + skipped_because_of: None, + }, + RunChange::Finished { + outcome: RunOutcome::Success, + reason: "staging completed".into(), + exit_code: Some(0), + }, + ] { + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: first.request.run_id, + claim_id: Some(first.claim_id), + change, + }, + ) + .await + .expect("staging fact"); + } + input.generation = 2; + input.documents.get_mut("jobs").expect("jobs")["marts"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("groups").expect("groups")["daily_orders"]["selection"] = + json!(["customers", "inventory"]); + publish_snapshot(store, &input) + .await + .expect("redeploy different membership"); + let second = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim marts") + .expect("marts ready"); + assert_eq!(second.request.snapshot_id, published.snapshot_id); + assert_eq!(second.request.selection, ["orders".to_owned()].into()); + assert_eq!(second.request.write_set, second.request.selection); + assert!( + run_execution(store, (second.request.run_id, second.claim_id)) + .await + .expect("execution") + .expect("pinned") + .skipped_nodes + .is_empty() + ); + let authority = RunAuthority::Worker { + run_id: second.request.run_id, + claim_id: second.claim_id, + }; + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: second.request.run_id, + claim_id: Some(second.claim_id), + change: RunChange::Started, + }, + ) + .await + .expect("start marts"); + change_run( + store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: request.run_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: "Cancel the group".into(), + }, + }, + ) + .await + .expect("cancel parent"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("propagate cancellation"); + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("parent") + .status, + RunStatus::Running, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == second.request.run_id) + .expect("active child") + .status, + RunStatus::Running + ); + let standalone = JobRunRequest { + run_id: Uuid::now_v7(), + job_key: "staging".into(), + ..request.clone() + }; + request_job_run(store, &context, &standalone) + .await + .expect("standalone request"); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("parent retains its slot") + .is_none() + ); + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: second.request.run_id, + claim_id: Some(second.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Cancelled, + reason: "worker confirmed its executor stopped".into(), + exit_code: Some(143), + }, + }, + ) + .await + .expect("physical child finished"); + let fault = TransactionFault { + description: "group terminal publication rolls back", + table: "recall_events", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected group publication failure')", + postgres_body: "RAISE EXCEPTION 'injected group publication failure'", + }; + install_fault(store, &fault).await; + assert!( + advance_groups(store, &lease, (&limits, scope)) + .await + .is_err() + ); + remove_fault(store, &fault).await; + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("retry group completion"); + } + let runs = list_runs(store, scope, 100).await.expect("completed runs"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("parent") + .status, + test_case.expected_status + ); + let standalone_claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("slot released") + .expect("standalone staging ready"); + assert_eq!(standalone_claim.request.run_id, standalone.run_id); + assert_eq!(standalone_claim.request.parent_run_id, None); + let events = read_recall_events(store, 0, 1000).await.expect("facts"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_terminated" + && event["invocation_id"] == format!("scheduler/{}", request.run_id)) + .count(), + 1 + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups_finish_without_exceeding_limits() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite opposing group reservations", + backend: Backend::Sqlite, + expected_status: RunStatus::Running, + }, + RunLifecycleTestCase { + description: "PostgreSQL opposing group reservations", + backend: Backend::Postgres, + expected_status: RunStatus::Running, + }, + ]; + let mut completed = Vec::new(); + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders_job": {"description": "Build orders", "selection": ["orders"], "concurrency_groups": ["inventory_limit"]}, + "inventory_job": {"description": "Build inventory", "selection": ["customers"], "concurrency_groups": ["orders_limit"]} + })); + input.documents.insert("groups".into(), json!({ + "orders_group": {"description": "Orders group", "selection": ["orders"], "concurrency_groups": ["orders_limit"], "jobs": {"orders_job": []}}, + "inventory_group": {"description": "Inventory group", "selection": ["customers"], "concurrency_groups": ["inventory_limit"], "jobs": {"inventory_job": []}} + })); + publish_snapshot(store, &input) + .await + .expect("publish groups"); + for job_key in ["orders_group", "inventory_group"] { + request_job_run( + store, + &context, + &JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: job_key.into(), + reason: "Build independent groups".into(), + priority: 1, + window: None, + }, + ) + .await + .expect("request group"); + } + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ("orders_limit".into(), NonZeroU32::new(1).expect("one slot")), + ( + "inventory_limit".into(), + NonZeroU32::new(1).expect("one slot"), + ), + ]); + let scope = (location, "development"); + for _ in 0..2 { + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("advance groups"); + } + let runs = list_runs(store, scope, 100).await.expect("runs"); + assert_eq!(runs.len(), 4, "{}", test_case.description); + assert_eq!( + runs.iter() + .filter(|run| run.status == test_case.expected_status) + .count(), + 1, + "{}", + test_case.description + ); + for _ in 0..2 { + let child = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim child") + .expect("a group must make progress"); + let slots = slot_counts(store).await; + assert_eq!( + slots.len(), + 2, + "child-only slot is acquired only for its worker" + ); + assert!(slots.values().all(|occupied| *occupied == 1), "{slots:?}"); + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("held child") + .is_none() + ); + succeed(store, &child).await; + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("complete parent and retry waiting group"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("retry independent of run-id ordering"); + } + let runs = list_runs(store, scope, 100).await.expect("completed runs"); + assert!( + runs.iter().all(|run| run.status == RunStatus::Success), + "{runs:?}" + ); + completed.push(test_case.description); + close_test_store(fixture).await; + }); + } + assert_eq!(completed.len(), 2, "both backends must run"); +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs new file mode 100644 index 0000000000..21ec54e845 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -0,0 +1,725 @@ +use std::collections::BTreeMap; + +use crate::credentials::models::AuditContext; +use crate::platform::models::PlatformEvent; +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_run_nodes::list_run_nodes; +use crate::runs::main::list_runs::list_runs; +use crate::runs::models::{ + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_facts_outcomes_and_claims_are_atomic() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite preserves facts, outcomes and claims through worker failures", + backend: Backend::Sqlite, + expected_status: RunStatus::Partial, + }, + RunLifecycleTestCase { + description: "PostgreSQL preserves facts, outcomes and claims through worker failures", + backend: Backend::Postgres, + expected_status: RunStatus::Partial, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + queue(store, &request(scope, 1, 2, &["customers", "orders"])).await; + queue(store, &request(scope, 2, 1, &["customers", "orders"])).await; + let limits = BTreeMap::new(); + let claim = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("available"); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let started = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Started, + }; + let other_worker = RunAuthority::Worker { + run_id: Uuid::from_u128(2), + claim_id: Uuid::now_v7(), + }; + let operator = AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!(change_run(store, other_worker, &started).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &started) + .await + .is_err() + ); + assert!( + change_run(store, worker, &started) + .await + .expect("worker can start before launcher acknowledgement") + ); + assert!( + !change_run(store, worker, &started) + .await + .expect("identical retry") + ); + let launched = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Launched { + container_id: Some("synthetic-orders-worker".into()), + }, + ..started.clone() + }; + change_run(store, RunAuthority::Scheduler(&lease), &launched) + .await + .expect("late launcher acknowledgement"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .iter() + .find(|run| run.run_id == claim.request.run_id) + .expect("started run") + .status, + RunStatus::Running + ); + let launch_failed = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::LaunchFailed { + code: "docker_launch_failed".into(), + }, + ..started.clone() + }; + assert!(change_run(store, worker, &launch_failed).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &launch_failed) + .await + .is_err() + ); + let wrong_claim = RunChangeRequest { + claim_id: Some(Uuid::now_v7()), + ..launch_failed.clone() + }; + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &wrong_claim) + .await + .is_err() + ); + let before_launch_failure = read_recall_events(store, 0, 100) + .await + .expect("before launch failure"); + let fault = TransactionFault { + description: "platform launch observation must commit with its run fact", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='platform_launch_failed'", + sqlite_body: "SELECT RAISE(ABORT,'injected launch observation failure')", + postgres_body: "RAISE EXCEPTION 'injected launch observation failure'", + }; + install_fault(store, &fault).await; + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back launch fact"), + before_launch_failure + ); + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .expect("launch observation") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(&lease), &launch_failed) + .await + .expect("idempotent retry") + ); + let observations: Vec<_> = read_recall_events(store, 0, 100) + .await + .expect("health facts") + .into_iter() + .filter(|row| row["event_type"] == "platform_launch_failed") + .collect(); + assert_eq!(observations.len(), 1); + let platform: PlatformEvent = + serde_json::from_value(observations[0]["payload"]["platform"].clone()) + .expect("typed launch failure"); + assert_eq!( + platform, + PlatformEvent::LaunchFailed { + run_id: started.run_id, + claim_id: claim.claim_id, + code: "docker_launch_failed".into() + } + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("live run") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run") + .status, + RunStatus::Running + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("physical ownership retained") + .is_none() + ); + let success = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: RunOutcome::Success, + reason: "worker complete".into(), + exit_code: Some(0), + }, + ..started.clone() + }; + assert!( + change_run(store, worker, &success).await.is_err(), + "unfinished nodes cannot be marked successful" + ); + for (node, outcome) in [ + ("customers", NodeOutcome::Success), + ("orders", NodeOutcome::Failed), + ] { + let node_start = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::NodeStarted { + node_key: node.into(), + }, + ..started.clone() + }; + change_run(store, worker, &node_start) + .await + .expect("node started"); + let completed = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::NodeFinished { + node_key: node.into(), + outcome, + rows_affected: Some(17), + error_code: (outcome == NodeOutcome::Failed).then(|| "E100".into()), + skipped_because_of: None, + }, + ..started.clone() + }; + let before_nodes = list_run_nodes(store, started.run_id) + .await + .expect("node before fault"); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("events before node fault"); + for table in ["sqb_run_nodes", "sqb_runs"] { + let point = TransactionFault { + description: "node projection and counter failure", + table, + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &completed).await.is_err(), + "{} at {table}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_run_nodes(store, started.run_id) + .await + .expect("node rollback"), + before_nodes + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("node fact rollback"), + before_events + ); + } + assert!( + change_run(store, worker, &completed) + .await + .expect("node completed") + ); + assert!( + !change_run(store, worker, &completed) + .await + .expect("node report retry") + ); + } + assert!( + change_run(store, worker, &success).await.is_err(), + "mixed node outcomes are partial, not success" + ); + let partial = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Finished { + outcome: RunOutcome::Partial, + reason: "independent customer work completed".into(), + exit_code: Some(1), + }, + ..started.clone() + }; + let before = read_recall_events(store, 0, 100) + .await + .expect("before terminal"); + for (table, operation) in [ + ("recall_events", "INSERT"), + ("sqb_runs", "UPDATE"), + ("sqb_write_claims", "DELETE"), + ("sqb_run_liveness", "DELETE"), + ] { + let point = TransactionFault { + description: "terminal transaction failure", + table, + operation, + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected terminal failure')", + postgres_body: "RAISE EXCEPTION 'injected terminal failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, worker, &partial).await.is_err(), + "{} at {table} must roll back the complete transition", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back event"), + before + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("claims retained") + .is_none() + ); + } + let unresponsive = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::Unresponsive, + ..started.clone() + }; + change_run(store, RunAuthority::Scheduler(&lease), &unresponsive) + .await + .expect("monitor records loss of heartbeat"); + assert!( + !heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .expect("heartbeat recovers the worker") + ); + assert!( + heartbeat_run(store, (started.run_id, Uuid::now_v7())) + .await + .is_err(), + "another claim cannot report this run alive" + ); + let cancel = RunChangeRequest { + event_id: Uuid::now_v7(), + change: RunChange::CancelRequested { + actor: "test-operator".into(), + reason: "Stop the orders build".into(), + }, + ..started.clone() + }; + assert!( + change_run(store, worker, &cancel).await.is_err(), + "workers cannot impersonate operators" + ); + change_run(store, RunAuthority::Operator(&operator), &cancel) + .await + .expect("record cancellation"); + assert!( + heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .expect("heartbeat reports pending cancellation") + ); + assert!( + claim_next_run(store, &lease, &limits, None) + .await + .expect("cancel request does not release an executing run's claims") + .is_none() + ); + assert!( + change_run(store, worker, &partial) + .await + .expect("terminal report atomically releases claims") + ); + assert!( + !change_run(store, worker, &partial) + .await + .expect("terminal retry after liveness cleanup") + ); + let mut changed = partial.clone(); + changed.change = RunChange::Finished { + outcome: RunOutcome::Success, + reason: "changed retry".into(), + exit_code: Some(0), + }; + assert!( + change_run(store, worker, &changed).await.is_err(), + "same event ID cannot change its content" + ); + assert!( + heartbeat_run(store, (started.run_id, claim.claim_id)) + .await + .is_err(), + "terminal runs cannot be revived by a late heartbeat" + ); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("terminal projection") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run"); + assert_eq!( + projected.status, test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + projected.node_counts_json, + BTreeMap::from([("success".into(), 1), ("failed".into(), 1)]) + ); + let nodes = list_run_nodes(store, started.run_id) + .await + .expect("node evidence"); + assert_eq!(nodes[0].node_key, "customers"); + assert_eq!(nodes[0].status, "success"); + assert_eq!(nodes[0].rows_affected, Some(17)); + assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); + let late_failure = RunChangeRequest { + event_id: Uuid::now_v7(), + ..launch_failed.clone() + }; + let before_late = read_recall_events(store, 0, 100) + .await + .expect("terminal facts"); + let expiry = TransactionFault { + description: "late launch observation loses leadership before commit", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ..fault + }; + for (point, fenced) in [(&fault, false), (&expiry, true)] { + install_fault(store, point).await; + let failed = + change_run(store, RunAuthority::Scheduler(&lease), &late_failure).await; + let error = failed.expect_err(point.description); + assert_eq!( + matches!(error, ControlError::Fenced), + fenced, + "{}", + point.description + ); + remove_fault(store, point).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("terminal rollback"), + before_late + ); + } + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &late_failure) + .await + .expect("late observation") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(&lease), &late_failure) + .await + .expect("late retry") + ); + assert!(change_run(store, worker, &late_failure).await.is_err()); + assert!( + change_run(store, RunAuthority::Operator(&operator), &late_failure) + .await + .is_err() + ); + for invalid in [ + RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: Some(Uuid::now_v7()), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + claim_id: None, + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(2), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(2), + claim_id: Some(Uuid::new_v5(&Uuid::from_u128(2), b"run_claimed")), + ..late_failure.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + ..started.clone() + }, + RunChangeRequest { + event_id: Uuid::now_v7(), + ..launched.clone() + }, + ] { + assert!( + change_run(store, RunAuthority::Scheduler(&lease), &invalid) + .await + .is_err(), + "terminal authority remains narrow: {invalid:?}" + ); + } + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("unchanged terminal projection") + .into_iter() + .find(|run| run.run_id == started.run_id) + .expect("original run"), + projected + ); + let recorded = read_recall_events(store, 0, 100) + .await + .expect("late fact pair"); + assert_eq!( + recorded + .iter() + .filter(|row| row["event_type"] == "platform_launch_failed") + .count(), + 2 + ); + expire_lease(store).await; + assert!( + matches!( + change_run(store, RunAuthority::Scheduler(&lease), &late_failure).await, + Err(ControlError::Fenced) + ), + "even duplicate observations require a live leader" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect successor") + .expect("successor"); + let following = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim following run") + .expect("released capacity"); + assert_eq!(following.request.run_id, Uuid::from_u128(2)); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_unclaimed_work_when_an_operator_cancels_then_a_terminal_fact_prevents_launch_and_preserves_the_actor() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite cancellation before launch preserves the operator and terminal fact", + backend: Backend::Sqlite, + expected_status: RunStatus::Cancelled, + }, + RunLifecycleTestCase { + description: "PostgreSQL cancellation before launch preserves the operator and terminal fact", + backend: Backend::Postgres, + expected_status: RunStatus::Cancelled, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + queue( + store, + &request((location, published.snapshot_id), 1, 0, &["orders"]), + ) + .await; + let request = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: Uuid::from_u128(1), + claim_id: None, + change: RunChange::CancelRequested { + actor: "test-operator".into(), + reason: "Cancel queued orders".into(), + }, + }; + let operator = AuditContext { + actor: "test-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let other = AuditContext { + actor: "another-operator".into(), + ..operator.clone() + }; + assert!( + change_run(store, RunAuthority::Operator(&other), &request) + .await + .is_err() + ); + let before_events = read_recall_events(store, 0, 100) + .await + .expect("facts before audit failure"); + let point = TransactionFault { + description: "cancellation audit failure rolls back terminal facts and projections", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected cancellation audit failure')", + postgres_body: "RAISE EXCEPTION 'injected cancellation audit failure'", + }; + install_fault(store, &point).await; + assert!( + change_run(store, RunAuthority::Operator(&operator), &request) + .await + .is_err(), + "{}", + point.description + ); + remove_fault(store, &point).await; + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("cancel rollback") + .remove(0) + .status, + RunStatus::Queued + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fact rollback"), + before_events + ); + assert!( + change_run(store, RunAuthority::Operator(&operator), &request) + .await + .expect("cancel before launch") + ); + assert!( + !change_run(store, RunAuthority::Operator(&operator), &request) + .await + .expect("retry cancellation") + ); + let mut changed = request.clone(); + changed.change = RunChange::CancelRequested { + actor: operator.actor.clone(), + reason: "Different cancellation reason".into(), + }; + assert!( + change_run(store, RunAuthority::Operator(&operator), &changed) + .await + .is_err(), + "retry identity includes the cancellation reason" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + assert!( + claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("cancelled work cannot launch") + .is_none() + ); + let projected = list_runs(store, (location, "development"), 100) + .await + .expect("cancelled projection") + .remove(0); + assert_eq!( + projected.status, test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + projected.node_counts_json, + BTreeMap::from([("cancelled".into(), 1)]) + ); + let events = read_recall_events(store, 0, 100) + .await + .expect("immutable cancellation facts"); + assert_eq!( + events + .iter() + .map(|row| row["event_type"].as_str().expect("type")) + .collect::>(), + [ + "run_requested", + "run_queued", + "run_cancel_requested", + "run_terminated" + ] + ); + assert_eq!( + events[2]["payload"]["change"]["change"]["actor"], + "test-operator" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs new file mode 100644 index 0000000000..62fe549bc8 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_monitor.rs @@ -0,0 +1,342 @@ +//! Real scheduler cycles detect silent workers without surrendering their physical write claims. + +use std::collections::BTreeMap; +use std::time::Duration; + +use crate::runs::main::change_run::change_run; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::heartbeat_run::heartbeat_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::monitor_runs::monitor_runs; +use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus}; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::contracts::helpers::{acquire_lease, scheduler_cycle}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::{queue, request, snapshot}; +use crate::runs::tests::monitor_helpers::{age_heartbeat, start_run}; +use crate::runs::tests::test_types::{ + MonitorBacklogTestCase, RunMonitorTestCase, TransactionFault, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_silent_and_live_workers_when_the_scheduler_monitors_then_fenced_atomic_facts_retain_claims_and_allow_recovery() + { + let test_cases = [ + RunMonitorTestCase { + description: "SQLite observes only running workers with expired heartbeats", + backend: Backend::Sqlite, + expected_statuses: [ + RunStatus::Unresponsive, + RunStatus::Running, + RunStatus::Launched, + RunStatus::Failed, + RunStatus::Queued, + ], + }, + RunMonitorTestCase { + description: "PostgreSQL observes only running workers with expired heartbeats", + backend: Backend::Postgres, + expected_statuses: [ + RunStatus::Unresponsive, + RunStatus::Running, + RunStatus::Launched, + RunStatus::Failed, + RunStatus::Queued, + ], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let scope = (location, published.snapshot_id); + let holder = Uuid::now_v7(); + let lease = acquire_lease(store, holder) + .await + .expect("elect") + .expect("leader"); + let silent = start_run(store, &lease, &request(scope, 1, 0, &["orders"])).await; + start_run(store, &lease, &request(scope, 2, 0, &["customers"])).await; + age_heartbeat(store, Uuid::from_u128(2), 40).await; + queue(store, &request(scope, 3, 0, &["inventory"])).await; + let launching = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim") + .expect("launching"); + change_run( + store, + RunAuthority::Scheduler(&lease), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: launching.request.run_id, + claim_id: Some(launching.claim_id), + change: RunChange::Launched { + container_id: Some("orders-worker".into()), + }, + }, + ) + .await + .expect("launcher acknowledges"); + let failed = start_run(store, &lease, &request(scope, 4, 0, &["products"])).await; + change_run( + store, + RunAuthority::Worker { + run_id: failed.request.run_id, + claim_id: failed.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: failed.request.run_id, + claim_id: Some(failed.claim_id), + change: RunChange::Finished { + outcome: RunOutcome::Failed, + reason: "worker failed".into(), + exit_code: Some(1), + }, + }, + ) + .await + .expect("terminal run"); + queue(store, &request(scope, 5, 0, &["orders"])).await; + age_heartbeat(store, silent.request.run_id, 50).await; + age_heartbeat(store, launching.request.run_id, 50).await; + let before = read_recall_events(store, 0, 100) + .await + .expect("before monitoring"); + let projection = list_runs(store, (location, "development"), 100) + .await + .expect("before monitoring"); + let fault = TransactionFault { + description: "monitor projection failure rolls back the observation", + table: "sqb_runs", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected monitor failure')", + postgres_body: "RAISE EXCEPTION 'injected monitor failure'", + }; + install_fault(store, &fault).await; + assert!( + monitor_runs(store, &lease).await.is_err(), + "{}", + test_case.description + ); + remove_fault(store, &fault).await; + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fact rollback"), + before + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("projection rollback"), + projection + ); + expire_lease(store).await; + let (state, observed) = scheduler_cycle(store, SchedulerState::new(holder)) + .await + .expect("monitoring cycle"); + assert_eq!(observed.unresponsive, 1, "{}", test_case.description); + let statuses: BTreeMap<_, _> = list_runs(store, (location, "development"), 100) + .await + .expect("monitored statuses") + .into_iter() + .map(|record| (record.run_id, record.status)) + .collect(); + assert_eq!( + std::array::from_fn(|index| statuses[&Uuid::from_u128(index as u128 + 1)]), + test_case.expected_statuses, + "{}", + test_case.description + ); + let recorded = read_recall_events(store, 0, 100) + .await + .expect("observation"); + assert_eq!(recorded.len(), before.len() + 1); + assert_eq!( + recorded.last().expect("fact")["event_type"], + "run_unresponsive" + ); + let (state, repeated) = scheduler_cycle(store, state) + .await + .expect("repeated monitor"); + assert_eq!(repeated.unresponsive, 0); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("no repeated facts"), + recorded + ); + let (_, follower) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("follower"); + assert!(!follower.leader); + assert_eq!(follower.unresponsive, 0); + assert!( + !heartbeat_run(store, (silent.request.run_id, silent.claim_id)) + .await + .expect("worker recovers") + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("recovered projection") + .into_iter() + .find(|record| record.run_id == silent.request.run_id) + .expect("recovered run") + .status, + RunStatus::Running + ); + age_heartbeat(store, silent.request.run_id, 50).await; + let (_, second_loss) = scheduler_cycle(store, state) + .await + .expect("new heartbeat loss"); + assert_eq!(second_loss.unresponsive, 1); + let events = read_recall_events(store, 0, 100) + .await + .expect("successive losses"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_unresponsive") + .count(), + 2 + ); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_recovered") + .count(), + 1 + ); + expire_lease(store).await; + assert!(matches!( + monitor_runs(store, &lease).await, + Err(ControlError::Fenced) + )); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("fenced monitor"), + events + ); + let successor = acquire_lease(store, Uuid::now_v7()) + .await + .expect("takeover") + .expect("successor"); + assert!( + claim_next_run(store, &successor, &BTreeMap::new(), None) + .await + .expect("write claims remain held after heartbeat loss and takeover") + .is_none() + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_silent_worker_backlog_when_monitoring_then_bounded_transactions_continue_immediately_until_drained() + { + let test_cases = [ + MonitorBacklogTestCase { + description: "SQLite drains more than one monitor page", + backend: Backend::Sqlite, + run_count: 105, + expected_batches: [100, 5, 0], + }, + MonitorBacklogTestCase { + description: "PostgreSQL drains more than one monitor page", + backend: Backend::Postgres, + run_count: 105, + expected_batches: [100, 5, 0], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let published = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let holder = Uuid::now_v7(); + let mut lease = acquire_lease(store, holder) + .await + .expect("elect") + .expect("leader"); + for index in 0..test_case.run_count { + let mut input = request( + (location, published.snapshot_id), + index as u128 + 1, + 0, + &["orders"], + ); + input.write_set.clear(); + start_run(store, &lease, &input).await; + age_heartbeat(store, input.run_id, 50).await; + lease = renew_lease(store, &lease) + .await + .expect("renew during setup"); + } + expire_lease(store).await; + let mut state = SchedulerState::new(holder); + let mut batches = Vec::new(); + for expected in test_case.expected_batches { + let cycle; + (state, cycle) = scheduler_cycle(store, state).await.expect("monitor batch"); + batches.push(cycle.unresponsive); + assert_eq!( + cycle.wait_for, + if expected == 100 { + Duration::ZERO + } else { + Duration::from_secs(5) + } + ); + } + assert_eq!( + batches, test_case.expected_batches, + "{}", + test_case.description + ); + let events = read_recall_events(store, 0, 1000) + .await + .expect("monitor facts"); + assert_eq!( + events + .iter() + .filter(|event| event["event_type"] == "run_unresponsive") + .count(), + test_case.run_count + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs new file mode 100644 index 0000000000..df6994d8cf --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -0,0 +1,249 @@ +//! Published-job requests freeze execution inputs and audit atomically on both databases. + +use std::collections::BTreeMap; + +use crate::credentials::models::AuditContext; +use crate::runs::main::claim_next_run::claim_next_run; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::{JobRunRequest, RunOrigin, RunStatus, RunWindow}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::helpers::read_recall_events; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{RunLifecycleTestCase, TransactionFault}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_and_frozen_execution_commit_once() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite queues and audits the pinned job exactly once", + backend: Backend::Sqlite, + expected_status: RunStatus::Queued, + }, + RunLifecycleTestCase { + description: "PostgreSQL queues and audits the pinned job exactly once", + backend: Backend::Postgres, + expected_status: RunStatus::Queued, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location_id); + let mut source = input.nodes["orders"].clone(); + source.kind = "source".into(); + input.nodes.insert("warehouse_orders".into(), source); + input.documents.insert("jobs".into(), json!({"daily_orders": { + "description": "Build orders from the warehouse source", "selection": ["orders", "warehouse_orders"], + "settings": {"full_refresh": true}, "concurrency_groups": ["orders-load"], "overlap": "queue" + }})); + let published = publish_snapshot(store, &input).await.expect("publish job"); + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Refresh orders".into(), + priority: 7, + window: Some(RunWindow::Timestamp { + start: "2026-07-03T01:17:23+05:30".parse().expect("start"), + end: "2026-07-04T01:17:23+05:30".parse().expect("end"), + }), + }; + for invalid in [ + JobRunRequest { + target: "production".into(), + ..request.clone() + }, + JobRunRequest { + location_id: Uuid::now_v7(), + ..request.clone() + }, + JobRunRequest { + job_key: "unknown_job".into(), + ..request.clone() + }, + JobRunRequest { + reason: " ".into(), + ..request.clone() + }, + JobRunRequest { + window: Some(RunWindow::Integer { start: 11, end: 3 }), + ..request.clone() + }, + JobRunRequest { + window: Some(RunWindow::Timestamp { + start: "2026-07-04T01:17:23Z".parse().expect("start"), + end: "2026-07-03T01:17:23Z".parse().expect("end"), + }), + ..request.clone() + }, + ] { + assert!(request_job_run(store, &context, &invalid).await.is_err()); + } + let fault = TransactionFault { + description: "manual run audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!(request_job_run(store, &context, &request).await.is_err()); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location_id, "development"), 10) + .await + .expect("rolled back projection") + .is_empty() + ); + assert!( + read_recall_events(store, 0, 100) + .await + .expect("rolled back facts") + .is_empty() + ); + let (first, retry) = tokio::join!( + request_job_run(store, &context, &request), + request_job_run(store, &context, &request) + ); + assert_ne!( + first.expect("first request"), + retry.expect("concurrent retry") + ); + let facts = read_recall_events(store, 0, 100) + .await + .expect("request facts"); + assert_eq!(facts.len(), 2); + input.generation = 2; + input.image_id = Some("sha256:synthetic-orders-image-2".into()); + input.documents.get_mut("jobs").expect("jobs")["daily_orders"]["selection"] = + json!(["inventory"]); + input.documents.get_mut("jobs").expect("jobs")["daily_orders"]["settings"] = + json!({"full_refresh": false}); + publish_snapshot(store, &input).await.expect("redeploy"); + assert!( + !request_job_run(store, &context, &request) + .await + .expect("retry keeps original snapshot") + ); + let mut changed = request.clone(); + changed.reason = "Different request".into(); + assert!(request_job_run(store, &context, &changed).await.is_err()); + changed = request.clone(); + changed.window = Some(RunWindow::Integer { start: 3, end: 11 }); + assert!(request_job_run(store, &context, &changed).await.is_err()); + let other_actor = AuditContext { + actor: "other-operator".into(), + ..context.clone() + }; + assert!( + request_job_run(store, &other_actor, &request) + .await + .is_err() + ); + assert_eq!( + read_recall_events(store, 0, 100) + .await + .expect("no duplicate facts"), + facts + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([( + "orders-load".into(), + std::num::NonZeroU32::new(1).expect("positive group limit"), + )]); + let claim = claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("queued request"); + assert_eq!(claim.request.run_id, request.run_id); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!(claim.request.origin, RunOrigin::Backfill); + assert_eq!( + claim.request.window, + Some(RunWindow::Timestamp { + start: "2026-07-02T19:47:23Z".parse().expect("normalized start"), + end: "2026-07-03T19:47:23Z".parse().expect("normalized end"), + }) + ); + assert_eq!( + claim.request.image_id.as_deref(), + Some("sha256:synthetic-orders-image") + ); + assert_eq!( + claim.request.selection, + ["orders".into(), "warehouse_orders".into()].into() + ); + assert_eq!(claim.request.write_set, ["orders".into()].into()); + assert_eq!( + claim.request.settings, + BTreeMap::from([("full_refresh".into(), json!(true))]) + ); + assert_eq!( + claim.request.concurrency_groups, + ["orders-load".into()].into() + ); + assert_eq!(claim.request.priority, 7); + assert_eq!(claim.request.requested_by, "test-operator"); + let fresh = JobRunRequest { + run_id: Uuid::now_v7(), + window: Some(RunWindow::Integer { start: 7, end: 7 }), + ..request.clone() + }; + assert!( + request_job_run(store, &context, &fresh) + .await + .expect("new request uses redeploy") + ); + let runs = list_runs(store, (location_id, "development"), 10) + .await + .expect("queued jobs"); + assert_eq!(runs.len(), 2); + assert_eq!( + runs.iter() + .find(|run| run.run_id == fresh.run_id) + .expect("new run") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == fresh.run_id) + .expect("new run") + .selection_json, + ["inventory".into()].into() + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs new file mode 100644 index 0000000000..75339ea1fe --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -0,0 +1,332 @@ +//! Terminal retries retain their snapshot and independent limits without repeating successful work. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::credentials::models::AuditContext; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + list_runs::list_runs, request_job_run::request_job_run, request_rerun::request_rerun, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RerunRequest, RunAuthority, RunChange, RunChangeRequest, RunOrigin, + RunOutcome, RunStatus, RunWindow, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::claim_helpers::{slot_counts, succeed}; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::{ + RerunExecutionTestCase, RunLifecycleTestCase, TransactionFault, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_nodes_retry_atomically() + { + let test_cases = [ + RunLifecycleTestCase { + description: "SQLite pinned group rerun", + backend: Backend::Sqlite, + expected_status: RunStatus::Success, + }, + RunLifecycleTestCase { + description: "PostgreSQL pinned group rerun", + backend: Backend::Postgres, + expected_status: RunStatus::Success, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, + "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, + "completed": {"description": "Build products", "selection": ["products"]} + })); + input.documents.insert("groups".into(), json!({"daily_orders": { + "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} + }})); + let published = publish_snapshot(store, &input).await.expect("publish"); + let source_id = Uuid::now_v7(); + request_job_run( + store, + &context, + &JobRunRequest { + run_id: source_id, + location_id: location, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build orders".into(), + priority: 1, + window: Some(RunWindow::Integer { start: -3, end: 11 }), + }, + ) + .await + .expect("request group"); + let retry = RerunRequest { + run_id: Uuid::from_u128(1), + source_run_id: source_id, + from_failure: true, + reason: "Retry failed orders".into(), + priority: 7, + }; + assert!( + request_rerun(store, &context, &retry).await.is_err(), + "a live source cannot be rerun" + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::from([ + ( + "warehouse".into(), + std::num::NonZeroU32::new(1).expect("positive capacity"), + ), + ( + "inventory_limit".into(), + std::num::NonZeroU32::new(1).expect("positive capacity"), + ), + ]); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit source"); + let node_outcomes = BTreeMap::from([ + ("products", NodeOutcome::Success), + ("orders", NodeOutcome::Failed), + ("customers", NodeOutcome::Cancelled), + ("inventory", NodeOutcome::Skipped), + ]); + let job_outcomes = BTreeMap::from([ + ("staging", RunOutcome::Failed), + ("completed", RunOutcome::Success), + ("marts", RunOutcome::Failed), + ]); + for _ in 0..3 { + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim source") + .expect("ready child"); + let mut changes = vec![RunChange::Started]; + for node in &claim.request.selection { + let outcome = node_outcomes[node.as_str()]; + changes.push(RunChange::NodeFinished { + node_key: node.clone(), + outcome, + rows_affected: None, + error_code: None, + skipped_because_of: (outcome == NodeOutcome::Skipped) + .then(|| "orders".into()), + }); + } + changes.push(RunChange::Finished { + outcome: job_outcomes[claim.request.job_key.as_deref().expect("source job")], + reason: "synthetic source outcome".into(), + exit_code: Some(1), + }); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("source worker fact"); + } + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish source"); + assert_eq!( + list_runs(store, scope, 20) + .await + .expect("list completed source") + .iter() + .find(|run| run.run_id == source_id) + .expect("source exists") + .status, + RunStatus::Partial + ); + input.generation = 2; + input.image_id = Some("sha256:synthetic-orders-image-2".into()); + input.documents.get_mut("jobs").expect("job declarations")["staging"]["settings"] = + json!({"vars": {"batch": 91}}); + publish_snapshot(store, &input).await.expect("redeploy"); + let fault = TransactionFault { + description: "rerun audit failure", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }; + install_fault(store, &fault).await; + assert!(request_rerun(store, &context, &retry).await.is_err()); + remove_fault(store, &fault).await; + assert_eq!( + list_runs(store, scope, 20) + .await + .expect("list rolled-back runs") + .len(), + 4, + "audit failure rolls back parent and all children" + ); + let (first, again) = tokio::join!( + request_rerun(store, &context, &retry), + request_rerun(store, &context, &retry) + ); + assert_ne!(first.expect("rerun"), again.expect("replica retry")); + let changed = RerunRequest { + from_failure: false, + ..retry.clone() + }; + assert!(request_rerun(store, &context, &changed).await.is_err()); + let other = AuditContext { + actor: "other-operator".into(), + ..context.clone() + }; + assert!(request_rerun(store, &other, &retry).await.is_err()); + let settings = BTreeMap::from([ + ( + "staging", + BTreeMap::from([("vars".into(), json!({"batch": 23}))]), + ), + ("completed", BTreeMap::new()), + ("marts", BTreeMap::new()), + ]); + for execution in [ + RerunExecutionTestCase { + description: "failure retry omits successful and cancelled nodes", + request: retry.clone(), + expected_applied: false, + expected_selection: ["orders", "inventory"].map(str::to_owned).into(), + expected_jobs: &["staging", "marts"], + }, + RerunExecutionTestCase { + description: "full retry retains all original nodes", + request: RerunRequest { + run_id: Uuid::now_v7(), + from_failure: false, + ..retry.clone() + }, + expected_applied: true, + expected_selection: ["orders", "inventory", "customers", "products"] + .map(str::to_owned) + .into(), + expected_jobs: &["staging", "completed", "marts"], + }, + ] { + let request = execution.request; + assert_eq!( + request_rerun(store, &context, &request) + .await + .expect("publish rerun"), + execution.expected_applied, + "{}", + execution.description + ); + let runs = list_runs(store, scope, 20).await.expect("list reruns"); + assert_eq!( + runs.iter() + .find(|run| run.run_id == request.run_id) + .expect("rerun exists") + .selection_json, + execution.expected_selection + ); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit rerun"); + let mut selected: BTreeSet = BTreeSet::new(); + for expected_job in execution.expected_jobs { + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim rerun") + .expect("ready retry child"); + assert_eq!( + claim.request.job_key.as_deref(), + Some(*expected_job), + "a completed intermediate job must not erase its ancestor's wait boundary" + ); + assert_eq!(claim.request.snapshot_id, published.snapshot_id); + assert_eq!( + claim.request.image_id, + Some("sha256:synthetic-orders-image".into()) + ); + assert_eq!(claim.request.origin, RunOrigin::Rerun); + assert_eq!(claim.request.priority, 7); + assert_eq!(claim.request.settings, settings[expected_job]); + assert_eq!( + claim.request.window, + Some(RunWindow::Integer { start: -3, end: 11 }) + ); + selected.extend(claim.request.selection.iter().cloned()); + succeed(store, &claim).await; + } + assert_eq!(selected, execution.expected_selection); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish rerun"); + assert_eq!( + list_runs(store, scope, 20) + .await + .expect("list finished reruns") + .iter() + .find(|run| run.run_id == request.run_id) + .expect("finished rerun exists") + .status, + test_case.expected_status, + "{}", + test_case.description + ); + assert!(slot_counts(store).await.is_empty()); + assert!( + !request_rerun(store, &context, &request) + .await + .expect("completed transport retry") + ); + let successful = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: request.run_id, + from_failure: true, + ..request.clone() + }; + assert!( + request_rerun(store, &context, &successful).await.is_err(), + "successful work has nothing to retry from failure" + ); + } + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs new file mode 100644 index 0000000000..fc108dd8c5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -0,0 +1,102 @@ +//! Expected queue outcomes for each real control-store backend. + +use std::collections::BTreeSet; + +use crate::runs::models::{RerunRequest, RunStatus}; + +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct ScopedClaimTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_selected_run: u128, + pub(crate) expected_unscoped_run: u128, +} + +pub(crate) struct PriorityClaimsTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_initial_claims: [u128; 2], + pub(crate) expected_claim_events: usize, + pub(crate) expected_terminal_events: usize, +} + +pub(crate) struct GroupLimitTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_unblocked_run: u128, +} + +pub(crate) struct BacklogClaimTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_independent_run: u128, +} + +pub(crate) struct CoalescedRunTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, +} + +pub(crate) struct ClaimFailureTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_run: u128, +} + +pub(crate) struct RunLifecycleTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_status: RunStatus, +} + +pub(crate) struct RerunExecutionTestCase { + pub(crate) description: &'static str, + pub(crate) request: RerunRequest, + pub(crate) expected_applied: bool, + pub(crate) expected_selection: BTreeSet, + pub(crate) expected_jobs: &'static [&'static str], +} + +pub(crate) struct RunMonitorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_statuses: [RunStatus; 5], +} + +pub(crate) struct MonitorBacklogTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) run_count: usize, + pub(crate) expected_batches: [usize; 3], +} + +pub(crate) struct AdmissionParent { + pub(crate) holds: &'static [&'static str], + pub(crate) children: &'static [&'static [&'static str]], +} + +pub(crate) struct GroupAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) limits: &'static [(&'static str, u32)], + pub(crate) parents: &'static [AdmissionParent], + pub(crate) expected_running: usize, + pub(crate) reductions: &'static [&'static [(&'static str, u32)]], +} + +pub(crate) struct GroupQueueTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) group_count: usize, + pub(crate) node_count: usize, + pub(crate) expected_max_claim_millis: u128, +} + +pub(crate) struct MixedAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) physical_priority: i64, + pub(crate) parent_priority: i64, + pub(crate) expected_selections: [&'static str; 2], +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs new file mode 100644 index 0000000000..a1e8ee33dc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -0,0 +1,72 @@ +//! Contention and expiry exercise scheduler ownership against real databases, not fake clocks. + +pub(crate) use crate::scheduler::main::_acquire_lease::acquire_lease; +pub(crate) use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +pub(crate) use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::renew_lease::renew_lease; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +pub(crate) async fn leadership_contract(backend: Backend) -> Option { + let test_store = open_test_store(backend).await?; + let store = &test_store.store; + migrate_store(store) + .await + .expect("migrate leadership store"); + let second = second_store(store).await; + let (first, other) = tokio::join!( + acquire_lease(store, Uuid::now_v7()), + acquire_lease(&second, Uuid::now_v7()) + ); + let first = first.expect("first contender completes"); + let other = other.expect("second contender completes"); + assert_ne!( + first.is_some(), + other.is_some(), + "exactly one concurrent contender is elected" + ); + let lease = first.or(other).expect("one elected holder"); + let renewed = renew_lease(&second, &lease) + .await + .expect("live epoch renews"); + assert_eq!(renewed.epoch, lease.epoch); + assert!(renewed.expires_at >= lease.expires_at); + assert!( + acquire_lease(store, lease.holder) + .await + .expect("duplicate acquisition completes") + .is_none() + ); + let mut forged = lease.clone(); + forged.holder = Uuid::now_v7(); + assert!(matches!( + renew_lease(store, &forged).await, + Err(ControlError::Fenced) + )); + expire_lease(store).await; + assert!( + matches!(renew_lease(store, &lease).await, Err(ControlError::Fenced)), + "even an uncontested expired holder must reacquire a new epoch" + ); + let successor = acquire_lease(&second, Uuid::now_v7()) + .await + .expect("acquire expired lease") + .expect("successor elected"); + assert!(matches!( + renew_lease(store, &lease).await, + Err(ControlError::Fenced) + )); + renew_lease(store, &successor) + .await + .expect("successor owns the current fence"); + let epoch = successor.epoch; + close_store(second).await; + close_test_store(test_store).await; + Some(epoch) +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs new file mode 100644 index 0000000000..13bdfd4250 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contract.rs @@ -0,0 +1,28 @@ +use crate::scheduler::tests::contracts::helpers::leadership_contract; +use crate::scheduler::tests::contracts::test_types::LeadershipTestCase; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_two_schedulers_when_leadership_expires_then_only_the_new_epoch_can_renew() { + let test_cases = [ + LeadershipTestCase { + description: "SQLite contenders and takeover", + backend: Backend::Sqlite, + expected_epoch: 2, + }, + LeadershipTestCase { + description: "PostgreSQL contenders and takeover", + backend: Backend::Postgres, + expected_epoch: 2, + }, + ]; + for test_case in test_cases { + let observed = run(leadership_contract(test_case.backend)); + assert!( + observed.is_none_or(|epoch| epoch == test_case.expected_epoch), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs new file mode 100644 index 0000000000..57d98cb7cf --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs @@ -0,0 +1,10 @@ +//! Real-store scheduler contracts compiled within the scheduler owner. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_types.rs"] +mod test_types; +#[path = "ticks/test_ticks.rs"] +mod ticks; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs new file mode 100644 index 0000000000..5259d9ee0b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_sensor_fairness.rs @@ -0,0 +1,128 @@ +//! Unavailable callback locations cannot monopolize the shared due-sensor page. + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::test_types::SensorFairnessTestCase; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{evaluations, snapshot}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_pending_custom_sensors_when_scheduling_then_rechecks_wait_and_other_locations_progress() { + let test_cases = [ + SensorFairnessTestCase { + description: "SQLite pending callback waits", + backend: Backend::Sqlite, + pending: 1, + builtin_status: "paused", + expected_completed: 0, + }, + SensorFairnessTestCase { + description: "PostgreSQL pending callback waits", + backend: Backend::Postgres, + pending: 1, + builtin_status: "paused", + expected_completed: 0, + }, + SensorFairnessTestCase { + description: "SQLite unavailable callback page permits builtin progress", + backend: Backend::Sqlite, + pending: 100, + builtin_status: "running", + expected_completed: 1, + }, + SensorFairnessTestCase { + description: "PostgreSQL unavailable callback page permits builtin progress", + backend: Backend::Postgres, + pending: 100, + builtin_status: "running", + expected_completed: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate fairness store"); + let custom = create_location(store, "fulfillment", "test-admin") + .await + .expect("callback location"); + let builtin = create_location(store, "reporting", "test-admin") + .await + .expect("builtin location"); + let mut input = snapshot(custom); + input.documents.insert("triggers".into(), json!({})); + for index in 0..test_case.pending { + input.documents.get_mut("triggers").expect("triggers")[format!("orders_{index}")] = json!({ + "description": "Poll fulfillment", "job_key": "orders", "when": {"kind": "custom", "function": "orders_ready", "code_hash": "orders_v1"}, + "minimum_interval_seconds": 1 + }); + } + publish_snapshot(store, &input) + .await + .expect("publish unavailable callbacks"); + publish_snapshot(store, &snapshot(builtin)) + .await + .expect("publish independent builtin"); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("set due order"); + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1 WHERE trigger_id IN (SELECT trigger_id FROM sqb_triggers WHERE location_id=$2)", + vec![Parameter::Time(Utc::now() - TimeDelta::hours(2)), Parameter::Id(custom)]).await.expect("callbacks first"); + let (transaction, _) = query(transaction, + "UPDATE sqb_trigger_state SET next_due_at=$1,status=$2 WHERE trigger_id IN (SELECT trigger_id FROM sqb_triggers WHERE location_id=$3)", + vec![Parameter::Time(Utc::now() - TimeDelta::hours(1)), Parameter::Text(test_case.builtin_status.into()), Parameter::Id(builtin)]).await.expect("builtin last"); + commit(transaction).await.expect("commit due order"); + let (state, first) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("pending page"); + assert_eq!( + first.evaluated, test_case.pending, + "{}", + test_case.description + ); + assert_eq!( + first.wait_for.is_zero(), + test_case.expected_completed == 1, + "{}: only genuinely overdue unprocessed work should require an immediate retry", + test_case.description + ); + scheduler_cycle(store, state).await.expect("next page"); + let facts = evaluations(store).await; + assert_eq!( + facts + .iter() + .filter(|fact| fact.outcome == SensorEvaluationOutcome::Requested) + .count(), + test_case.pending + test_case.expected_completed, + "{}", + test_case.description + ); + assert_eq!( + facts + .iter() + .filter(|fact| fact.outcome != SensorEvaluationOutcome::Requested) + .count(), + test_case.expected_completed, + "{}: builtin at another location completes without waiting for unavailable callbacks", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs new file mode 100644 index 0000000000..fd5543086b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -0,0 +1,10 @@ +//! Leadership scenarios run independently on both supported engines. + +use crate::store::tests::store_test_types::Backend; + +#[derive(Debug)] +pub(crate) struct LeadershipTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_epoch: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs new file mode 100644 index 0000000000..223bc36025 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/helpers.rs @@ -0,0 +1,290 @@ +//! Trigger and failure fixtures exercise the scheduler's transactional tick boundary. + +use std::collections::BTreeMap; + +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::scheduler::main::list_ticks::list_ticks; +use crate::scheduler::models::TickOutcome; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::ControlStore; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::test_types::TransactionFault; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::helpers::read_recall_events; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: false, + git_author: None, + image_id: Some("sha256:synthetic-orders-image".into()), + environment_hash: "orders-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table"}), + raw_sql: Some("select 1 as order_id".into()), + compiled_sql: Some("select 1 as order_id".into()), + python_source: None, + }, + )]), + documents: BTreeMap::from([ + ( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ), + ( + "triggers".into(), + json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": "2020-01-01T00:00:00Z", + "schedule": {"kind": "calendar", "times": ["2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", "2020-01-03T00:00:00Z", "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}}), + ), + ]), + } +} + +async fn trigger_audit_rows(store: &ControlStore) -> Vec<(String, String, String, String, String)> { + let sql = "SELECT CAST(event_id AS TEXT), actor, granting_role, CAST(before_json AS TEXT), CAST(after_json AS TEXT) FROM sqb_security_audit WHERE action='trigger.changed'"; + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return rusqlite::Connection::open(path) + .expect("inspect trigger audit") + .prepare(sql) + .expect("audit query") + .query_map([], |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }) + .expect("read trigger audits") + .collect::>() + .expect("audit rows"); + }; + sqlx_core::query_as::query_as(sql) + .fetch_all(pool) + .await + .expect("trigger audits") +} + +pub(crate) async fn trigger_audits(store: &ControlStore) -> Vec { + trigger_audit_rows(store) + .await + .into_iter() + .map(|(event_id, actor, granting_role, before, after)| { + json!({ + "event_id": event_id, + "actor": actor, + "granting_role": granting_role, + "before_json": serde_json::from_str::(&before).expect("before state"), + "after_json": serde_json::from_str::(&after).expect("after state"), + }) + }) + .collect() +} + +pub(crate) async fn transaction_contract(backend: Backend) -> Option<(usize, usize)> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store).await.expect("migrate tick store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish job and trigger"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("published triggers")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("acquire lease") + .expect("elected"); + let second = second_store(store).await; + for point in [ + TransactionFault { + description: "tick insertion", + table: "sqb_ticks", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected tick failure')", + postgres_body: "RAISE EXCEPTION 'injected tick failure'", + }, + TransactionFault { + description: "requested event", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='run_requested'", + sqlite_body: "SELECT RAISE(ABORT,'injected requested failure')", + postgres_body: "RAISE EXCEPTION 'injected requested failure'", + }, + TransactionFault { + description: "queued event", + table: "recall_events", + operation: "INSERT", + condition: "WHEN NEW.event_type='run_queued'", + sqlite_body: "SELECT RAISE(ABORT,'injected queued failure')", + postgres_body: "RAISE EXCEPTION 'injected queued failure'", + }, + TransactionFault { + description: "run projection", + table: "sqb_runs", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected run failure')", + postgres_body: "RAISE EXCEPTION 'injected run failure'", + }, + TransactionFault { + description: "node projection", + table: "sqb_run_nodes", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected node failure')", + postgres_body: "RAISE EXCEPTION 'injected node failure'", + }, + TransactionFault { + description: "cursor commit", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected cursor failure'", + }, + TransactionFault { + description: "lease expires during append", + table: "sqb_runs", + operation: "INSERT", + condition: "", + sqlite_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + postgres_body: "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + }, + ] { + install_fault(store, &point).await; + let failed = evaluate_trigger(store, &lease, trigger_id).await; + assert!(failed.is_err(), "{} must abort", point.description); + remove_fault(store, &point).await; + assert!( + list_ticks(&second, trigger_id, 100) + .await + .expect("rolled-back ticks") + .is_empty(), + "{}", + point.description + ); + assert!( + list_runs(&second, (location, "development"), 100) + .await + .expect("rolled-back runs") + .is_empty(), + "{}", + point.description + ); + assert!( + read_recall_events(&second, 0, 100) + .await + .expect("rolled-back events") + .is_empty(), + "{}", + point.description + ); + assert!( + list_triggers(&second, location, "development") + .await + .expect("rolled-back cursor")[0] + .last_tick_for + .is_none(), + "{}", + point.description + ); + } + let (left, right) = tokio::join!( + evaluate_trigger(store, &lease, trigger_id), + evaluate_trigger(&second, &lease, trigger_id) + ); + let first_count = left + .expect("first scheduler") + .expect("active trigger") + .actions + .len(); + let second_count = right + .expect("second scheduler") + .expect("active trigger") + .actions + .len(); + assert_eq!( + first_count + second_count, + 3, + "restarts never repeat committed ticks" + ); + let ticks = list_ticks(store, trigger_id, 100) + .await + .expect("committed ticks"); + let runs = list_runs(store, (location, "development"), 100) + .await + .expect("committed run"); + let events = read_recall_events(store, 0, 100) + .await + .expect("committed events"); + assert_eq!( + events + .iter() + .map(|row| row["event_type"].as_str().expect("event type")) + .collect::>(), + ["run_requested", "run_queued"] + ); + assert_eq!(ticks[0].outcome, TickOutcome::Requested); + assert!( + ticks[1..] + .iter() + .all(|tick| tick.outcome == TickOutcome::Covered + && tick.covered_by_run_id == Some(runs[0].run_id)) + ); + assert_eq!( + runs[0].last_event_storage_id, + events[1]["storage_id"].as_i64().expect("event cursor") + ); + expire_lease(store).await; + let successor = acquire_lease(&second, Uuid::now_v7()) + .await + .expect("takeover") + .expect("new leader"); + assert!(matches!( + evaluate_trigger(store, &lease, trigger_id).await, + Err(ControlError::Fenced) + )); + assert!( + evaluate_trigger(&second, &successor, trigger_id) + .await + .expect("failover catch-up") + .expect("active trigger") + .actions + .is_empty() + ); + let counts = (ticks.len(), runs.len()); + close_store(second).await; + close_test_store(fixture).await; + Some(counts) +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs new file mode 100644 index 0000000000..91d523b69f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_contract.rs @@ -0,0 +1,34 @@ +//! Crash points must not separate a tick from its queued work and durable cursor. + +use crate::scheduler::tests::contracts::ticks::helpers::transaction_contract; +use crate::scheduler::tests::contracts::ticks::test_types::TickTransactionTestCase; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_two_schedulers_and_faults_between_writes_when_restarting_then_ticks_runs_recall_and_cursor_commit_together() + { + let test_cases = [ + TickTransactionTestCase { + description: "SQLite fenced tick transaction", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL fenced tick transaction", + backend: Backend::Postgres, + expected_ticks: 3, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + let observed = run(transaction_contract(test_case.backend)); + assert!( + observed + .is_none_or(|counts| counts == (test_case.expected_ticks, test_case.expected_runs)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs new file mode 100644 index 0000000000..382d0f17bf --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_loop.rs @@ -0,0 +1,106 @@ +//! Embedded and standalone schedulers share takeover and shutdown semantics. + +use std::time::Duration; + +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::main::run_scheduler::run_scheduler; +use crate::scheduler::models::SchedulerState; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::scheduler::tests::contracts::ticks::test_types::{ + IdleLauncher, TickTransactionTestCase, +}; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_embedded_and_standalone_schedulers_when_taking_over_then_only_one_drains_due_work_and_idle_shutdown_is_prompt() + { + let test_cases = [ + TickTransactionTestCase { + description: "SQLite scheduler loop", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL scheduler loop", + backend: Backend::Postgres, + expected_ticks: 3, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate loop store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish due schedule"); + let second = second_store(store).await; + let first_state = SchedulerState::new(Uuid::now_v7()); + let second_state = SchedulerState::new(Uuid::now_v7()); + let (first_state, first) = scheduler_cycle(store, first_state) + .await + .expect("initial cycle"); + assert!(first.leader, "{}", test_case.description); + assert_eq!(first.evaluated, 1); + assert_eq!(first.ticks, test_case.expected_ticks); + assert_eq!(first.wait_for, Duration::from_secs(5)); + let (second_state, follower) = scheduler_cycle(&second, second_state) + .await + .expect("follower cycle"); + assert!(!follower.leader); + assert_eq!(follower.ticks, 0); + expire_lease(store).await; + let (second_state, successor) = scheduler_cycle(&second, second_state) + .await + .expect("takeover cycle"); + assert!(successor.leader); + assert_eq!(successor.ticks, 0, "takeover never repeats durable work"); + assert!( + !scheduler_cycle(store, first_state) + .await + .expect("fenced cycle") + .1 + .leader + ); + let runs = + crate::runs::main::list_runs::list_runs(store, (location, "development"), 100) + .await + .expect("queued run"); + assert_eq!(runs.len(), test_case.expected_runs); + let (stop, shutdown) = tokio::sync::watch::channel(false); + let (_changed, changes) = tokio::sync::watch::channel(()); + let mut running = Box::pin(run_scheduler( + &second, + second_state, + (shutdown, changes), + IdleLauncher, + )); + tokio::select! { + result = &mut running => panic!("loop ended before shutdown: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(50)) => {}, + } + stop.send(true).expect("signal shutdown"); + tokio::time::timeout(Duration::from_secs(2), running) + .await + .expect("idle shutdown does not wait for the five-second timer") + .expect("scheduler exits cleanly"); + close_store(second).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs new file mode 100644 index 0000000000..3f36d1d4fe --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_ticks.rs @@ -0,0 +1,10 @@ +#[path = "helpers.rs"] +mod helpers; +#[path = "test_contract.rs"] +mod test_contract; +#[path = "test_loop.rs"] +mod test_loop; +#[path = "test_triggers.rs"] +mod test_triggers; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs new file mode 100644 index 0000000000..41610c718b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_triggers.rs @@ -0,0 +1,372 @@ +//! Operator trigger facts remain consistent across publication and retry boundaries. + +use crate::credentials::models::AuditContext; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::main::list_trigger_events::list_trigger_events; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest, TriggerStatus}; +use crate::scheduler::main::_acquire_lease::acquire_lease; +use crate::scheduler::main::_evaluate_trigger::evaluate_trigger; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::scheduler::tests::contracts::ticks::helpers::{snapshot, trigger_audits}; +use crate::scheduler::tests::contracts::ticks::test_types::{ + TickTransactionTestCase, TransactionFault, TriggerControlsTestCase, +}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_operator_changes_when_retried_and_republished_then_facts_and_effective_state_remain_consistent() + { + let test_cases = [ + TriggerControlsTestCase { + description: "SQLite operator facts", + backend: Backend::Sqlite, + expected_events: 5, + }, + TriggerControlsTestCase { + description: "PostgreSQL operator facts", + backend: Backend::Postgres, + expected_events: 5, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate trigger store"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("initial publish"); + let trigger_id = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let pause = TriggerChangeRequest { + event_id: Uuid::now_v7(), + trigger_id, + actor: "orders-operator".into(), + reason: "maintenance".into(), + change: TriggerChange::Pause, + }; + let audit = AuditContext { + actor: "orders-operator".into(), + granting_role: "operator".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!(change_trigger(store, &audit, &pause).await.expect("pause")); + let paused = list_triggers(store, location, "development") + .await + .expect("paused trigger") + .remove(0); + assert_eq!( + paused.status, + TriggerStatus::Paused, + "{}", + test_case.description + ); + assert!(paused.next_due_at.is_none()); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + assert!( + evaluate_trigger(store, &lease, trigger_id) + .await + .expect("paused evaluation") + .is_none() + ); + let enable = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::Enable, + ..pause.clone() + }; + for fault in [ + TransactionFault { + description: "operator event and projection", + table: "sqb_trigger_state", + operation: "UPDATE", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected state failure')", + postgres_body: "RAISE EXCEPTION 'injected state failure'", + }, + TransactionFault { + description: "operator security audit", + table: "sqb_security_audit", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT,'injected audit failure')", + postgres_body: "RAISE EXCEPTION 'injected audit failure'", + }, + ] { + install_fault(store, &fault).await; + assert!(change_trigger(store, &audit, &enable).await.is_err()); + remove_fault(store, &fault).await; + assert_eq!( + list_trigger_events(store, trigger_id, 100) + .await + .expect("rolled-back fact") + .len(), + 1 + ); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("rolled back projection")[0] + .status, + TriggerStatus::Paused + ); + } + assert!( + change_trigger(store, &audit, &enable) + .await + .expect("retry enable") + ); + assert!( + !change_trigger(store, &audit, &pause) + .await + .expect("retry old pause is inert") + ); + let conflict = TriggerChangeRequest { + change: TriggerChange::ResetDefault, + ..pause.clone() + }; + assert!( + change_trigger(store, &audit, &conflict).await.is_err(), + "same event ID cannot change content" + ); + input.generation = 2; + input + .documents + .get_mut("triggers") + .expect("trigger document")["daily_orders"]["default_status"] = + serde_json::json!("paused"); + publish_snapshot(store, &input) + .await + .expect("publish a changed code default"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("retained override")[0] + .status, + TriggerStatus::Running + ); + let reset = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::ResetDefault, + ..pause.clone() + }; + change_trigger(store, &audit, &reset) + .await + .expect("restore new code default"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("default restored")[0] + .status, + TriggerStatus::Paused + ); + let time = chrono::DateTime::parse_from_rfc3339("2020-01-02T00:00:00Z") + .expect("cursor") + .to_utc(); + let cursor = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::SetCursor { time }, + ..pause.clone() + }; + change_trigger(store, &audit, &cursor) + .await + .expect("set durable cursor"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("cursor set")[0] + .last_tick_for, + Some(time) + ); + let reset_cursor = TriggerChangeRequest { + event_id: Uuid::now_v7(), + change: TriggerChange::ResetCursor, + ..pause.clone() + }; + change_trigger(store, &audit, &reset_cursor) + .await + .expect("reset cursor"); + assert!( + list_triggers(store, location, "development") + .await + .expect("cursor reset")[0] + .last_tick_for + .is_none() + ); + let events = list_trigger_events(store, trigger_id, 100) + .await + .expect("operator history"); + assert_eq!(events.len(), test_case.expected_events); + assert!( + events + .iter() + .all(|event| event.request_json.actor == "orders-operator") + ); + let other_actor = AuditContext { + actor: "customers-operator".into(), + ..audit.clone() + }; + assert!(change_trigger(store, &other_actor, &pause).await.is_err()); + let audits = trigger_audits(store).await; + assert_eq!(audits.len(), test_case.expected_events); + assert!( + audits + .iter() + .all(|row| row["actor"] == "orders-operator" + && row["granting_role"] == "operator") + ); + let cleared = audits + .iter() + .find(|row| row["event_id"] == reset_cursor.event_id.to_string()) + .expect("cursor reset audit"); + assert_eq!( + cleared["before_json"]["last_tick_for"], + "2020-01-02T00:00:00Z" + ); + assert!(cleared["after_json"]["last_tick_for"].is_null()); + assert_eq!(cleared["after_json"]["reason"], "maintenance"); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_calendar_with_unprocessed_due_times_when_refreshed_then_due_ticks_survive_and_only_future_times_are_replaced() + { + use crate::runs::main::list_runs::list_runs; + use crate::scheduler::main::list_ticks::list_ticks; + use serde_json::json; + + let test_cases = [ + TickTransactionTestCase { + description: "SQLite retains due calendar ticks", + backend: Backend::Sqlite, + expected_ticks: 3, + expected_runs: 1, + }, + TickTransactionTestCase { + description: "PostgreSQL retains due calendar ticks", + backend: Backend::Postgres, + expected_ticks: 3, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("publish pending calendar"); + input.generation = 2; + let future: chrono::DateTime = + "2031-01-01T00:00:00Z".parse().expect("future"); + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = json!( + (0..10_000) + .map(|seconds| future + chrono::TimeDelta::seconds(seconds)) + .collect::>() + ); + assert!( + publish_snapshot(store, &input) + .await + .expect_err("pending ticks cannot be truncated to fit a refreshed horizon") + .to_string() + .contains("let the scheduler drain pending ticks") + ); + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2031-01-01T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh while scheduler is offline"); + let trigger = list_triggers(store, location, "development") + .await + .expect("refreshed trigger") + .remove(0); + assert_eq!( + serde_json::to_value(&trigger.definition_json).expect("calendar")["schedule"]["times"], + json!([ + "2020-01-01T00:00:00Z", + "2020-01-02T00:00:00Z", + "2020-01-03T00:00:00Z", + "2031-01-01T00:00:00Z" + ]), + "{}", + test_case.description + ); + assert_eq!( + trigger.next_due_at, + Some("2020-01-01T00:00:00Z".parse().expect("due")) + ); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("election") + .expect("lease"); + let plan = evaluate_trigger(store, &lease, trigger.trigger_id) + .await + .expect("catch up") + .expect("trigger"); + assert_eq!(plan.actions.len(), test_case.expected_ticks); + assert_eq!( + list_ticks(store, trigger.trigger_id, 100) + .await + .expect("ticks") + .len(), + test_case.expected_ticks + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("coalesced run") + .len(), + test_case.expected_runs + ); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!(["2032-01-01T00:00:00Z"]); + publish_snapshot(store, &input) + .await + .expect("refresh after catch-up"); + let trigger = list_triggers(store, location, "development") + .await + .expect("drained trigger") + .remove(0); + assert_eq!( + serde_json::to_value(&trigger.definition_json).expect("calendar")["schedule"]["times"], + json!(["2032-01-01T00:00:00Z"]) + ); + assert_eq!( + list_ticks(store, trigger.trigger_id, 100) + .await + .expect("immutable tick history") + .len(), + test_case.expected_ticks + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs new file mode 100644 index 0000000000..d523cedb8e --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/ticks/test_types.rs @@ -0,0 +1,37 @@ +//! Database cases and expected scheduler tick outcomes. + +pub(crate) use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct TickTransactionTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_ticks: usize, + pub(crate) expected_runs: usize, +} + +pub(crate) struct TriggerControlsTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_events: usize, +} + +pub(crate) struct IdleLauncher; + +impl crate::scheduler::types::RunLauncher for IdleLauncher { + async fn poll( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + Ok(()) + } + + async fn shutdown( + &mut self, + _store: &crate::store::models::ControlStore, + _lease: Option<&crate::scheduler::models::SchedulerLease>, + ) -> Result<(), crate::store::errors::ControlError> { + Ok(()) + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs new file mode 100644 index 0000000000..ff6089837f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -0,0 +1,253 @@ +use chrono::{DateTime, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::declarations::main::change_trigger::change_trigger; +use crate::declarations::models::{TriggerChange, TriggerChangeRequest}; +use crate::platform::models::PlatformEvent; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::models::SchedulerLease; +use crate::sensors::models::SensorEvaluationFact; +use crate::snapshots::models::SnapshotInput; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::models::{ControlEvent, ControlStore, TransactionMode}; + +use crate::runs::tests::helpers::snapshot as base_snapshot; +use crate::store::tests::helpers::read_recall_events; + +pub(crate) fn snapshot(location_id: Uuid) -> SnapshotInput { + let mut input = base_snapshot(location_id); + input.documents.insert( + "jobs".into(), + json!({ + "orders": {"description": "Refresh orders", "selection": ["orders"], "overlap": "allow"} + }), + ); + input.documents.insert( + "triggers".into(), + json!({ + "health_orders": { + "description": "Refresh reporting on overdue ticks", "job_key": "orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"]}, + "minimum_interval_seconds": 1 + } + }), + ); + input +} + +pub(crate) async fn append_events( + store: &ControlStore, + indices: std::ops::Range, + occurred_at: DateTime, +) { + let mut transaction = begin(store, TransactionMode::Write) + .await + .expect("event transaction"); + let payloads = [ + PlatformEvent::TickOverdue { + trigger_id: Uuid::from_u128(7), + scheduled_for: occurred_at, + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(8), + }, + PlatformEvent::LocationBroken { + location_id: Uuid::from_u128(9), + target: "development".into(), + last_seen_at: occurred_at, + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(10), + }, + PlatformEvent::LaunchFailed { + run_id: Uuid::from_u128(11), + claim_id: Uuid::from_u128(12), + code: "local_spawn_failed".into(), + }, + PlatformEvent::Heartbeat { + replica_id: Uuid::from_u128(13), + }, + ]; + for index in indices { + let platform = &payloads[(index % 6) as usize]; + let (next, appended) = append_control_event( + transaction, + ControlEvent { + event_id: Uuid::from_u128(1000 + index), + invocation_id: "platform/test-health".into(), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: None, + payload: json!({"platform": platform}), + }, + ) + .await + .expect("append platform observation"); + assert!(appended.inserted); + transaction = next; + } + commit(transaction).await.expect("commit observations"); +} + +pub(crate) async fn evaluations(store: &ControlStore) -> Vec { + read_recall_events(store, 0, 1000) + .await + .expect("read durable history") + .into_iter() + .filter(|row| row["event_type"] == "sensor_evaluation") + .map(|row| serde_json::from_value(row["payload"].clone()).expect("typed attempt fact")) + .collect() +} + +pub(crate) async fn change(store: &ControlStore, trigger_id: Uuid, change: TriggerChange) { + let event_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: event_id, + ip_address: None, + }; + let request = TriggerChangeRequest { + event_id, + trigger_id, + actor: context.actor.clone(), + reason: "Test operator change".into(), + change, + }; + assert!( + change_trigger(store, &context, &request) + .await + .expect("apply operator fact") + ); + assert!( + !change_trigger(store, &context, &request) + .await + .expect("retry operator fact") + ); +} + +/// Real request, claim, node facts and finish, including the virtual group parent. +pub(crate) async fn finish_run( + store: &ControlStore, + lease: &SchedulerLease, + scope: (Uuid, &str), + job_key: &str, + outcome: RunOutcome, + nodes: [NodeOutcome; 2], + started: [bool; 2], +) -> Uuid { + let run_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id, + location_id: scope.0, + target: scope.1.into(), + job_key: job_key.into(), + reason: "Exercise terminal sensor scope".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request source run"); + finish_claimed_run(store, lease, scope, run_id, outcome, nodes, started).await; + run_id +} + +/// Execute only the frozen selection, including a from-failure rerun's reduced scope. +pub(crate) async fn finish_claimed_run( + store: &ControlStore, + lease: &SchedulerLease, + scope: (Uuid, &str), + run_id: Uuid, + outcome: RunOutcome, + nodes: [NodeOutcome; 2], + started: [bool; 2], +) { + let limits = std::collections::BTreeMap::new(); + advance_groups(store, lease, (&limits, scope)) + .await + .expect("advance parent"); + let claim = claim_next_run(store, lease, &limits, Some(scope)) + .await + .expect("claim source") + .expect("source ready"); + assert!(claim.request.run_id == run_id || claim.request.parent_run_id == Some(run_id)); + let worker = RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }; + let mut changes = vec![RunChange::Started]; + for (index, key) in ["customers", "orders"] + .into_iter() + .enumerate() + .filter(|(_, key)| claim.request.selection.contains(*key)) + { + changes.extend(started[index].then_some(RunChange::NodeStarted { + node_key: key.into(), + })); + changes.push(RunChange::NodeFinished { + node_key: key.into(), + outcome: nodes[index], + rows_affected: None, + error_code: None, + skipped_because_of: None, + }); + } + for change in changes { + change_run( + store, + worker, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("publish worker evidence"); + } + let terminal = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change: RunChange::Finished { + outcome, + reason: "Terminal outcome fixture".into(), + exit_code: None, + }, + }; + assert!( + change_run(store, RunAuthority::Scheduler(lease), &terminal) + .await + .expect("finish") + ); + assert!( + !change_run(store, RunAuthority::Scheduler(lease), &terminal) + .await + .expect("retry finish") + ); + advance_groups(store, lease, (&limits, scope)) + .await + .expect("complete parent"); +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs new file mode 100644 index 0000000000..84e211d230 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -0,0 +1,18 @@ +//! Real stores prove atomic sensor consumption and recovery across scheduler ownership changes. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_custom.rs"] +mod test_custom; +#[path = "test_delivery.rs"] +mod test_delivery; +#[path = "test_execution_bounds.rs"] +mod test_execution_bounds; +#[path = "test_nodes.rs"] +mod test_nodes; +#[path = "test_recovery.rs"] +mod test_recovery; +#[path = "test_runs.rs"] +mod test_runs; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs new file mode 100644 index 0000000000..e3407c4419 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs @@ -0,0 +1,443 @@ +//! The real outbound HTTP boundary returns data, never scheduler authority. + +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::helpers::{start_api, stop_api}; +use crate::credentials::main::issue_token::issue_token; +use crate::credentials::models::{AuditContext, CredentialScope}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::platform::main::connect_location_work::connect_location_work; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, LocationWork, LocationWorkOutcome, LocationWorkPoll, + LocationWorkResult, +}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_custom_sensor::complete_custom_sensor; +use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationOutcome}; +use crate::sensors::tests::helpers::{evaluations, snapshot}; +use crate::sensors::tests::test_types::{CustomSensorResultCase, SensorTestCase}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_custom_sensor_work_when_results_retry_then_cursor_runs_and_outcome_commit_atomically() { + let test_cases = [ + SensorTestCase { + description: "SQLite custom sensor protocol", + backend: Backend::Sqlite, + expected_runs: 2, + }, + SensorTestCase { + description: "PostgreSQL custom sensor protocol", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.get_mut("triggers").expect("triggers")["health_orders"]["when"] = + json!({"kind": "custom", "function": "orders_ready", "code_hash": "orders_v1"}); + publish_snapshot(store, &input) + .await + .expect("publish custom sensor data"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let token = issue_token( + store, + &context, + CredentialScope::Location { + location_id: location, + }, + Utc::now() + TimeDelta::hours(1), + ) + .await + .expect("scoped token"); + let api = start_api(store).await; + let client = + connect_location_work((&api.url, &token.bearer), location).expect("client"); + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: "development".into(), + }; + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: poll.session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register location instance"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let point = TransactionFault { + description: "request and outbound work are atomic", + table: "sqb_location_work", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor request failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor request failure'", + }; + install_fault(store, &point).await; + assert!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .is_err() + ); + remove_fault(store, &point).await; + assert!( + evaluations(store).await.is_empty(), + "request fact rolled back with work" + ); + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("begin") + .expect("due"); + assert_eq!( + complete_custom_sensor(store, &lease, &requested) + .await + .expect("still waiting"), + 0 + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "no invented terminal phase while waiting" + ); + let claim = client.poll(&poll).await.expect("poll").expect("work"); + assert_eq!(claim.request.request_id, requested.evaluation_id); + assert_eq!( + claim.request.work, + LocationWork::EvaluateSensor { + evaluation: Box::new(requested.clone()) + } + ); + assert_eq!( + client.poll(&poll).await.expect("lost poll response"), + Some(claim.clone()) + ); + let wrong = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: true }, + }; + assert!( + client + .complete(requested.evaluation_id, &wrong) + .await + .is_err(), + "wrong result kind" + ); + let empty_key = LocationWorkResult { + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("unsafe".into()), + run_keys: vec![" ".into()], + }, + }, + ..wrong.clone() + }; + assert!( + client + .complete(requested.evaluation_id, &empty_key) + .await + .is_err(), + "empty keys fail closed" + ); + let result = LocationWorkResult { + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("orders/page-2".into()), + run_keys: vec!["orders-2".into(), "orders-1".into(), "orders-2".into()], + }, + }, + ..wrong + }; + assert!( + client + .complete(requested.evaluation_id, &result) + .await + .expect("durable location result") + ); + assert!( + !client + .complete(requested.evaluation_id, &result) + .await + .expect("uncertain acknowledgement retry") + ); + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .is_empty(), + "location result alone has no queue authority" + ); + for table in [ + "sqb_runs", + "sqb_run_requests", + "sqb_sensor_evaluation_events", + "sqb_sensor_cursor_commits", + ] { + let point = TransactionFault { + description: "custom sensor atomic publication", + table, + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + install_fault(store, &point).await; + assert!( + complete_custom_sensor(store, &lease, &requested) + .await + .is_err() + ); + remove_fault(store, &point).await; + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("rollback runs") + .is_empty() + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "outcome rolled back with cursor and queue" + ); + assert_eq!( + begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("unchanged cursor"), + Some(requested.clone()) + ); + } + let replica = second_store(store).await; + assert_eq!( + complete_custom_sensor(&replica, &lease, &requested) + .await + .expect("replica commit"), + test_case.expected_runs + ); + assert_eq!( + complete_custom_sensor(store, &lease, &requested) + .await + .expect("retry commit"), + 0 + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("dedup runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + let facts = evaluations(store).await; + let SensorEvaluationOutcome::Completed { cursor, run_ids } = + &facts.last().expect("completed").outcome + else { + panic!("completed phase required"); + }; + assert_eq!(cursor.custom.as_deref(), Some("orders/page-2")); + assert_eq!(cursor.event_storage_id, requested.cursor.event_storage_id); + assert_eq!(run_ids.len(), test_case.expected_runs); + for result_case in [ + CustomSensorResultCase { + outcome: CustomSensorOutcome::Completed { + cursor: Some("orders/page-2".into()), + run_keys: vec!["orders-2".into(), "orders-1".into()], + }, + expected_phase: "completed", + expected_cursor: Some("orders/page-2"), + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::Failed, + expected_phase: "failed", + expected_cursor: None, + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::TimedOut, + expected_phase: "timed_out", + expected_cursor: None, + }, + CustomSensorResultCase { + outcome: CustomSensorOutcome::Skipped { + cursor: Some("orders/page-3".into()), + reason: "No new orders".into(), + }, + expected_phase: "completed", + expected_cursor: Some("orders/page-3"), + }, + ] { + tokio::time::sleep(Duration::from_secs(1)).await; + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next attempt") + .expect("due"); + assert_eq!( + pending.cursor.custom.as_deref(), + Some("orders/page-2"), + "failed and timeout leave cursor unchanged" + ); + let claim = client + .poll(&poll) + .await + .expect("next poll") + .expect("next work"); + assert_eq!(claim.request.request_id, pending.evaluation_id); + let result = LocationWorkResult { + claim_id: claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: result_case.outcome, + }, + }; + assert!( + client + .complete(pending.evaluation_id, &result) + .await + .expect("complete callback") + ); + complete_custom_sensor(store, &lease, &pending) + .await + .expect("apply phase"); + let facts = evaluations(store).await; + let outcome = &facts.last().expect("terminal evaluation phase").outcome; + assert_eq!(outcome.phase(), result_case.expected_phase); + let payload = serde_json::to_value(outcome).expect("outcome payload"); + assert_eq!( + payload["cursor"]["custom"].as_str(), + result_case.expected_cursor + ); + } + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("still deduped") + .len(), + test_case.expected_runs + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let stale = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request before takeover") + .expect("due"); + let stale_claim = client + .poll(&poll) + .await + .expect("claim before takeover") + .expect("work"); + assert_eq!(stale_claim.request.request_id, stale.evaluation_id); + expire_lease(store).await; + assert!( + client + .poll(&poll) + .await + .expect("no expired-epoch dispatch") + .is_none() + ); + assert!(matches!( + complete_custom_sensor(store, &lease, &stale).await, + Err(ControlError::Fenced) + )); + let successor = acquire_lease(&replica, Uuid::now_v7()) + .await + .expect("takeover") + .expect("leader"); + let late = LocationWorkResult { + claim_id: stale_claim.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::SensorEvaluation { + outcome: CustomSensorOutcome::Completed { + cursor: Some("uncommitted/stale".into()), + run_keys: vec!["must-not-queue".into()], + }, + }, + }; + assert!( + client + .complete(stale.evaluation_id, &late) + .await + .expect("late location result") + ); + assert!(matches!( + complete_custom_sensor(&replica, &successor, &stale).await, + Err(ControlError::Fenced) + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let recovered = begin_sensor_evaluation(&replica, &successor, trigger.trigger_id) + .await + .expect("recover abandoned callback") + .expect("due after minimum interval"); + assert_eq!(recovered.cursor.custom.as_deref(), Some("orders/page-3")); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("no stale queued request") + .len(), + test_case.expected_runs + ); + assert_eq!( + evaluations(store) + .await + .iter() + .filter(|fact| fact.request.evaluation_id == stale.evaluation_id + && fact.outcome == SensorEvaluationOutcome::Abandoned) + .count(), + 1 + ); + assert_eq!( + client + .poll(&poll) + .await + .expect("new epoch work") + .expect("new claim") + .request + .request_id, + recovered.evaluation_id, + "retired evaluation must not block or execute before current work" + ); + close_store(replica).await; + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs new file mode 100644 index 0000000000..de5581844b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs @@ -0,0 +1,183 @@ +use std::collections::BTreeSet; +use std::time::Duration; + +use chrono::{TimeDelta, Utc}; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_paged_platform_events_when_sensor_publication_fails_then_retry_is_atomic_and_reset_deduplicates() + { + let test_cases = [ + SensorTestCase { + description: "SQLite sensor delivery", + backend: Backend::Sqlite, + expected_runs: 54, + }, + SensorTestCase { + description: "PostgreSQL sensor delivery", + backend: Backend::Postgres, + expected_runs: 54, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let now = Utc::now(); + append_events(store, 0..105, now).await; + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + let replica = second_store(store).await; + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover pending"), + Some(requested.clone()) + ); + for table in [ + "sqb_runs", + "sqb_run_requests", + "sqb_sensor_evaluation_events", + "sqb_sensor_cursor_commits", + ] { + let point = TransactionFault { + description: "fail sensor transaction", + table, + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + install_fault(store, &point).await; + assert!( + complete_builtin_sensor(store, &lease, &requested) + .await + .is_err(), + "{table}" + ); + remove_fault(store, &point).await; + assert!( + list_runs(store, (location, "development"), 1000) + .await + .expect("rolled back runs") + .is_empty(), + "{table}" + ); + assert_eq!( + evaluations(store).await.len(), + 1, + "only requested survives {table}" + ); + } + assert_eq!( + complete_builtin_sensor(&replica, &lease, &requested) + .await + .expect("retry first page"), + 50 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &requested) + .await + .expect("retry completion"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let second = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next page") + .expect("due"); + assert!(second.cursor.event_storage_id > 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &second) + .await + .expect("remaining page"), + 3 + ); + // Committed later, but its occurrence precedes the entire first page. + append_events(store, 106..107, now - TimeDelta::days(1)).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let late = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("late request") + .expect("due"); + assert!(late.cursor.event_storage_id > second.cursor.event_storage_id); + assert_eq!( + complete_builtin_sensor(store, &lease, &late) + .await + .expect("late arrival"), + 1 + ); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + assert_eq!(replay.cursor.event_storage_id, 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("replay retained first page"), + 50 + ); + let runs = list_runs(store, (location, "development"), 1000) + .await + .expect("queued runs"); + assert_eq!( + runs.len(), + test_case.expected_runs, + "{}: reset never requeues an existing run key", + test_case.description + ); + assert!( + runs.iter() + .all(|run| run.selection_json == BTreeSet::from(["orders".into()])) + ); + let facts = evaluations(store).await; + assert_eq!(facts.len(), 8); + assert!(facts.iter().all(|fact| matches!( + fact.outcome, + SensorEvaluationOutcome::Requested | SensorEvaluationOutcome::Completed { .. } + ))); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs new file mode 100644 index 0000000000..3cc7469c46 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs @@ -0,0 +1,176 @@ +//! A wide group forces multiple predicate chunks; the last node is not optional. + +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::runs::main::{ + advance_groups::advance_groups, change_run::change_run, claim_next_run::claim_next_run, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::snapshot; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_wide_group_when_only_the_last_node_has_not_started_then_all_chunks_must_match_within_a_bounded_evaluation() + { + let test_cases = [ + SensorTestCase { + description: "SQLite wide executed-node predicate", + backend: Backend::Sqlite, + expected_runs: 1, + }, + SensorTestCase { + description: "PostgreSQL wide executed-node predicate", + backend: Backend::Postgres, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + let node = input.nodes["orders"].clone(); + let selection: Vec = (0..501) + .map(|index| format!("model:orders_{index:04}")) + .collect(); + input + .nodes + .extend(selection.iter().map(|key| (key.clone(), node.clone()))); + input.documents.insert("jobs".into(), json!({ + "source": {"description": "Wide source", "selection": selection, "overlap": "allow"}, + "destination": {"description": "Destination", "selection": ["orders"], "overlap": "allow"} + })); + input.documents.insert("groups".into(), json!({ + "wide_orders": {"description": "Wide group", "selection": selection, "jobs": {"source": []}, "overlap": "allow"} + })); + input.documents.insert("triggers".into(), json!({ + "executed": {"description": "All source nodes must execute", "job_key": "destination", + "when": {"kind": "run_event", "job_key": "wide_orders", "kinds": ["completed"], "executed_successfully": selection}, + "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input) + .await + .expect("publish wide predicate"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = std::collections::BTreeMap::new(); + let scope = (location, "development"); + for include_last in [true, false] { + let parent = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id: parent, + location_id: location, + target: "development".into(), + job_key: "wide_orders".into(), + reason: "Exercise wide execution evidence".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("admit parent"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("child"); + assert_eq!(claim.request.parent_run_id, Some(parent)); + let changes = std::iter::once(RunChange::Started) + .chain( + selection + .iter() + .filter(|key| include_last || key.as_str() != "model:orders_0500") + .map(|key| RunChange::NodeStarted { + node_key: key.clone(), + }), + ) + .chain(selection.iter().map(|key| RunChange::NodeFinished { + node_key: key.clone(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + })) + .chain(std::iter::once(RunChange::Finished { + outcome: RunOutcome::Success, + reason: "All logical nodes succeed".into(), + exit_code: None, + })); + for change in changes { + change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("real node evidence"); + } + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("finish parent"); + } + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let evaluation = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request evaluation") + .expect("due"); + let count = tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(store, &lease, &evaluation), + ) + .await + .expect("wide predicate must not hold the write lock unboundedly") + .expect("complete"); + assert_eq!(count, test_case.expected_runs, "{}", test_case.description); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs new file mode 100644 index 0000000000..bc2556174c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs @@ -0,0 +1,376 @@ +//! Real lifecycle facts exercise any-run rebuilds, early delivery and paged atomic recovery. + +use std::collections::BTreeMap; +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::{ + change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, + request_job_run::request_job_run, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, +}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_run, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queue_once_per_finish() +{ + let test_cases = [ + SensorTestCase { + description: "SQLite any-run node successes", + backend: Backend::Sqlite, + expected_runs: 8, + }, + SensorTestCase { + description: "PostgreSQL any-run node successes", + backend: Backend::Postgres, + expected_runs: 8, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required store"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store).await.expect("warm migration"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source orders", "selection": ["orders", "customers"], "overlap": "allow"}, + "inventory": {"description": "Sensor destination", "selection": ["orders", "customers"], "overlap": "allow"} + })); + input.documents.insert("groups".into(), json!({ + "daily_orders": {"description": "Source group", "selection": ["orders", "customers"], "jobs": {"orders": []}, "overlap": "allow"} + })); + input.documents.insert("triggers".into(), json!({ + "rebuilt_orders": {"description": "Each genuine rebuild", "job_key": "inventory", + "when": {"kind": "node_event", "selection": ["orders", "customers"], "kinds": ["success"]}, + "minimum_interval_seconds": 1} + })); + publish_snapshot(store, &input).await.expect("publish"); + let mut isolated = input.clone(); + isolated.location_id = other; + publish_snapshot(store, &isolated) + .await + .expect("other location snapshot"); + isolated.location_id = location; + isolated.target = "staging".into(); + publish_snapshot(store, &isolated) + .await + .expect("other target snapshot"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + for (outcome, nodes, started) in [ + (RunOutcome::Success, [NodeOutcome::Success; 2], [true; 2]), + (RunOutcome::Success, [NodeOutcome::Success; 2], [false; 2]), + ( + RunOutcome::Warn, + [NodeOutcome::Warn, NodeOutcome::Success], + [true; 2], + ), + (RunOutcome::Failed, [NodeOutcome::Failed; 2], [true; 2]), + ( + RunOutcome::Partial, + [NodeOutcome::Skipped, NodeOutcome::Success], + [false, true], + ), + ( + RunOutcome::Cancelled, + [NodeOutcome::Cancelled; 2], + [true; 2], + ), + ] { + finish_run( + store, + &lease, + (location, "development"), + "orders", + outcome, + nodes, + started, + ) + .await; + } + for scope in [(other, "development"), (location, "staging")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + } + finish_run( + store, + &lease, + (location, "development"), + "daily_orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + // Page past nonmatches without borrowing starts from earlier real runs. + for _ in 0..46 { + finish_run( + store, + &lease, + (location, "development"), + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + } + let run_id = Uuid::now_v7(); + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + request_job_run( + store, + &context, + &JobRunRequest { + run_id, + location_id: location, + target: "development".into(), + job_key: "orders".into(), + reason: "Deliver before whole-run completion".into(), + priority: 10, + window: None, + }, + ) + .await + .expect("request running source"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("claim") + .expect("ready"); + assert_eq!(claim.request.run_id, run_id); + let authority = RunAuthority::Worker { + run_id, + claim_id: claim.claim_id, + }; + let changes = [ + RunChange::Started, + RunChange::NodeStarted { + node_key: "orders".into(), + }, + RunChange::NodeFinished { + node_key: "orders".into(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ]; + for change in changes { + let request = RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change, + }; + let mut wrong = request.clone(); + wrong.claim_id = Some(Uuid::now_v7()); + assert!( + change_run(store, authority, &wrong).await.is_err(), + "another claim cannot supply start/finish evidence" + ); + assert!( + change_run(store, authority, &request) + .await + .expect("worker fact") + ); + assert!( + !change_run(store, authority, &request) + .await + .expect("duplicate worker fact") + ); + } + let pending = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request evaluation") + .expect("due"); + let replica = second_store(store).await; + let fault = TransactionFault { + description: "node cursor rollback", + table: "sqb_sensor_cursor_commits", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected node cursor failure')", + postgres_body: "RAISE EXCEPTION 'injected node cursor failure'", + }; + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &pending) + .await + .is_err() + ); + remove_fault(store, &fault).await; + assert!( + list_runs(store, (location, "development"), 1000) + .await + .expect("rollback") + .iter() + .all(|run| run.job_key.as_deref() != Some("inventory")) + ); + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover pending"), + Some(pending.clone()) + ); + assert_eq!( + tokio::time::timeout( + Duration::from_secs(5), + complete_builtin_sensor(&replica, &lease, &pending), + ) + .await + .expect("bounded 100-event page") + .expect("first page"), + 6 + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &pending) + .await + .expect("duplicate completion"), + 0 + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let next = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("second evaluation") + .expect("due"); + assert!(next.cursor.event_storage_id > 0); + assert_eq!( + complete_builtin_sensor(store, &lease, &next) + .await + .expect("second page"), + 1, + "the other selected node is still pending" + ); + for change in [ + RunChange::NodeStarted { + node_key: "customers".into(), + }, + RunChange::NodeFinished { + node_key: "customers".into(), + outcome: NodeOutcome::Success, + rows_affected: None, + error_code: None, + skipped_because_of: None, + }, + ] { + change_run( + store, + authority, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await + .expect("later node in the same run"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + let later = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("later evaluation") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &later) + .await + .expect("later finish"), + 1, + "distinct successful finishes in the same run must not collapse" + ); + let runs = list_runs(store, (location, "development"), 1000) + .await + .expect("runs"); + assert_eq!( + runs.iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs, + "{}", + test_case.description + ); + assert_eq!( + runs.iter() + .find(|run| run.run_id == run_id) + .expect("source retained") + .status, + crate::runs::models::RunStatus::Running + ); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + for _ in 0..2 { + tokio::time::sleep(Duration::from_secs(1)).await; + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("replay") + .expect("due"); + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("deduplicated replay"); + } + assert_eq!( + list_runs(store, (location, "development"), 1000) + .await + .expect("replayed runs") + .iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .count(), + test_case.expected_runs + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs new file mode 100644 index 0000000000..426fc861a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_recovery.rs @@ -0,0 +1,218 @@ +use std::time::Duration; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::{TriggerChange, TriggerStatus}; +use crate::runs::main::list_runs::list_runs; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::models::SensorEvaluationOutcome; +use crate::sensors::tests::helpers::{append_events, change, evaluations, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_crashed_sensor_when_leadership_or_snapshot_changes_then_stale_results_cannot_queue_runs() +{ + let test_cases = [ + SensorTestCase { + description: "SQLite sensor recovery", + backend: Backend::Sqlite, + expected_runs: 1, + }, + SensorTestCase { + description: "PostgreSQL sensor recovery", + backend: Backend::Postgres, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + publish_snapshot(store, &input) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + append_events(store, 0..1, chrono::Utc::now()).await; + let first = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let pending = begin_sensor_evaluation(store, &first, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + expire_lease(store).await; + let replica = second_store(store).await; + let successor = acquire_lease(&replica, Uuid::now_v7()) + .await + .expect("takeover") + .expect("new leader"); + assert!(matches!( + complete_builtin_sensor(store, &first, &pending).await, + Err(ControlError::Fenced) + )); + assert!( + begin_sensor_evaluation(&replica, &successor, trigger.trigger_id) + .await + .expect("recover abandoned attempt") + .is_none(), + "a crash still counts toward the minimum interval" + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("abandoned fact") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let requested = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("new request") + .expect("due"); + assert_ne!(pending.evaluation_id, requested.evaluation_id); + let mut forged = requested.clone(); + forged.cursor.event_storage_id = 999; + assert!(matches!( + complete_builtin_sensor(store, &successor, &forged).await, + Err(ControlError::Fenced) + )); + input.generation += 1; + publish_snapshot(store, &input).await.expect("redeploy"); + assert_eq!( + complete_builtin_sensor(store, &successor, &requested) + .await + .expect("abandon stale snapshot"), + 0 + ); + assert!( + list_runs(store, (location, "development"), 100) + .await + .expect("no stale runs") + .is_empty() + ); + change(store, trigger.trigger_id, TriggerChange::Pause).await; + input.generation += 1; + publish_snapshot(store, &input) + .await + .expect("redeploy paused sensor"); + assert_eq!( + list_triggers(store, location, "development") + .await + .expect("preserved override")[0] + .status, + TriggerStatus::Paused + ); + assert!( + begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("paused") + .is_none() + ); + change(store, trigger.trigger_id, TriggerChange::ResetDefault).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let current = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("code default") + .expect("running"); + assert_eq!( + complete_builtin_sensor(store, &successor, ¤t) + .await + .expect("catch up retained health"), + test_case.expected_runs, + "{}", + test_case.description + ); + tokio::time::sleep(Duration::from_secs(1)).await; + let before_reset = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request before reset") + .expect("due"); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + assert_eq!( + complete_builtin_sensor(store, &successor, &before_reset) + .await + .expect("reset fences old cursor"), + 0 + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("reset abandonment") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + tokio::time::sleep(Duration::from_secs(1)).await; + let before_pause = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request before pause") + .expect("due"); + change(store, trigger.trigger_id, TriggerChange::Pause).await; + assert_eq!( + complete_builtin_sensor(store, &successor, &before_pause) + .await + .expect("pause fences pending result"), + 0 + ); + assert!(matches!( + evaluations(store) + .await + .last() + .expect("pause abandonment") + .outcome, + SensorEvaluationOutcome::Abandoned + )); + change(store, trigger.trigger_id, TriggerChange::ResetDefault).await; + tokio::time::sleep(Duration::from_secs(1)).await; + let delayed = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("request delayed evaluation") + .expect("due"); + tokio::time::sleep(Duration::from_secs(1)).await; + complete_builtin_sensor(store, &successor, &delayed) + .await + .expect("complete delayed evaluation"); + let immediate = begin_sensor_evaluation(store, &successor, trigger.trigger_id) + .await + .expect("interval measured from start, not completion") + .expect("already due"); + assert_ne!(immediate.evaluation_id, delayed.evaluation_id); + complete_builtin_sensor(store, &successor, &immediate) + .await + .expect("complete next attempt"); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("deduplicated runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs new file mode 100644 index 0000000000..44a6f6f588 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -0,0 +1,550 @@ +//! Actual lifecycle facts feed scoped outcome selectors and atomic recovery on both stores. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_triggers::list_triggers; +use crate::declarations::models::TriggerChange; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::request_rerun::request_rerun; +use crate::runs::models::{NodeOutcome, RerunRequest, RunOutcome}; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::{ + begin_sensor_evaluation::begin_sensor_evaluation, + complete_builtin_sensor::complete_builtin_sensor, +}; +use crate::sensors::tests::helpers::{change, finish_claimed_run, finish_run, snapshot}; +use crate::sensors::tests::test_types::{SensorExecutionTestCase, SensorTestCase}; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{ + begin::begin, commit::commit, migrate_store::migrate_store, query::query, +}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::fault_helpers::{install_fault, remove_fault}; +use crate::store::tests::fault_types::TransactionFault; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_run_keys_are_preserved() + { + let test_cases = [ + SensorTestCase { + description: "SQLite terminal outcome sensor recovery", + backend: Backend::Sqlite, + expected_runs: 106, + }, + SensorTestCase { + description: "PostgreSQL terminal outcome sensor recovery", + backend: Backend::Postgres, + expected_runs: 106, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + migrate_store(store) + .await + .expect("warm migration preserves index"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "products", "test-admin") + .await + .expect("other location"); + let mut input = snapshot(location); + input.documents.insert("jobs".into(), json!({ + "orders": {"description": "Source orders", "selection": ["orders", "customers"], "overlap": "allow"}, + "inventory": {"description": "Sensor destination", "selection": ["orders", "customers"], "overlap": "allow"} + })); + input.documents.insert("groups".into(), json!({ + "daily_orders": {"description": "Source group", "selection": ["orders", "customers"], "jobs": {"orders": []}, "overlap": "allow"} + })); + let selectors = [ + "completed", + "success", + "warn", + "partial", + "failed", + "cancelled", + "presumed_failed", + ]; + let mut sensors = serde_json::Map::new(); + for job_key in ["orders", "daily_orders"] { + for selector in selectors { + sensors.insert(format!("{job_key}_{selector}"), json!({ + "description": "Consume exact terminal outcomes", "job_key": "inventory", + "when": {"kind": "run_event", "job_key": job_key, "kinds": [selector]}, + "minimum_interval_seconds": 1 + })); + } + for (name, kind, nodes) in [ + ("both_completed", "completed", vec!["customers", "orders"]), + ("customers_completed", "completed", vec!["customers"]), + ("customers_success", "success", vec!["customers"]), + ("customers_partial", "partial", vec!["customers"]), + ("orders_completed", "completed", vec!["orders"]), + ] { + sensors.insert(format!("{job_key}_executed_{name}"), json!({ + "description": "Require every selected node to actually succeed", "job_key": "inventory", + "when": {"kind": "run_event", "job_key": job_key, "kinds": [kind], "executed_successfully": nodes}, + "minimum_interval_seconds": 1 + })); + } + } + input.documents.insert("triggers".into(), json!(sensors)); + publish_snapshot(store, &input) + .await + .expect("publish sensors"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut expected: BTreeMap> = BTreeMap::new(); + let outcomes = [ + ( + "success", + RunOutcome::Success, + [NodeOutcome::Success, NodeOutcome::Success], + ), + ( + "warn", + RunOutcome::Warn, + [NodeOutcome::Warn, NodeOutcome::Success], + ), + ( + "partial", + RunOutcome::Partial, + [NodeOutcome::Success, NodeOutcome::Failed], + ), + ( + "failed", + RunOutcome::Failed, + [NodeOutcome::Failed, NodeOutcome::Failed], + ), + ( + "cancelled", + RunOutcome::Cancelled, + [NodeOutcome::Cancelled, NodeOutcome::Cancelled], + ), + ( + "presumed_failed", + RunOutcome::PresumedFailed, + [NodeOutcome::Failed, NodeOutcome::Failed], + ), + ]; + for (selector, outcome, nodes) in outcomes { + let id = finish_run( + store, + &lease, + (location, "development"), + "orders", + outcome, + nodes, + [false; 2], + ) + .await + .to_string(); + expected + .entry(format!("orders_{selector}")) + .or_default() + .insert(id.clone()); + expected + .entry("orders_completed".into()) + .or_default() + .insert(id); + } + let group = finish_run( + store, + &lease, + (location, "development"), + "daily_orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + let sources = list_runs(store, (location, "development"), 100) + .await + .expect("sources"); + let child = sources + .iter() + .find(|record| { + record.run_id != group + && record.job_key.as_deref() == Some("orders") + && !expected["orders_completed"].contains(&record.run_id.to_string()) + }) + .expect("group child"); + for selector in ["completed", "success"] { + expected + .entry(format!("daily_orders_{selector}")) + .or_default() + .insert(group.to_string()); + expected + .entry(format!("orders_{selector}")) + .or_default() + .insert(child.run_id.to_string()); + } + let executions = [ + SensorExecutionTestCase { + description: "freshness-only success cannot borrow another execution's starts", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [false; 2], + expected_selector: "success", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "all selected nodes must start, not merely any node", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [true, false], + expected_selector: "success", + expected_filters: &["customers_completed", "customers_success"], + }, + SensorExecutionTestCase { + description: "both selected nodes actually succeed", + outcome: RunOutcome::Success, + nodes: [NodeOutcome::Success; 2], + started: [true; 2], + expected_selector: "success", + expected_filters: &[ + "both_completed", + "customers_completed", + "customers_success", + "orders_completed", + ], + }, + SensorExecutionTestCase { + description: "partial outcome is independent of selected-node success", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Success, NodeOutcome::Failed], + started: [true; 2], + expected_selector: "partial", + expected_filters: &["customers_completed", "customers_partial"], + }, + SensorExecutionTestCase { + description: "warning is not successful execution", + outcome: RunOutcome::Warn, + nodes: [NodeOutcome::Warn, NodeOutcome::Success], + started: [true; 2], + expected_selector: "warn", + expected_filters: &["orders_completed"], + }, + SensorExecutionTestCase { + description: "start without success cannot borrow success from another run", + outcome: RunOutcome::Failed, + nodes: [NodeOutcome::Failed; 2], + started: [true; 2], + expected_selector: "failed", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "skipped selected node is not successful", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Skipped, NodeOutcome::Success], + started: [false, true], + expected_selector: "partial", + expected_filters: &["orders_completed"], + }, + SensorExecutionTestCase { + description: "a partial run's freshness-only node still does not qualify", + outcome: RunOutcome::Partial, + nodes: [NodeOutcome::Success, NodeOutcome::Failed], + started: [false, true], + expected_selector: "partial", + expected_filters: &[], + }, + SensorExecutionTestCase { + description: "a genuinely started warning is not success", + outcome: RunOutcome::Warn, + nodes: [NodeOutcome::Warn, NodeOutcome::Success], + started: [true, false], + expected_selector: "warn", + expected_filters: &[], + }, + ]; + let mut source_ids: BTreeMap<(&str, &str), Uuid> = BTreeMap::new(); + for execution in executions { + for job_key in ["orders", "daily_orders"] { + let id = finish_run( + store, + &lease, + (location, "development"), + job_key, + execution.outcome, + execution.nodes, + execution.started, + ) + .await; + source_ids.insert((execution.description, job_key), id); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read lineage"); + let (transaction, children) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE parent_run_id=$1", + vec![Parameter::Id(id)], + ) + .await + .expect(execution.description); + commit(transaction).await.expect("read lineage committed"); + for (source, id) in std::iter::once((job_key, id.to_string())).chain( + children.into_iter().map(|row| { + ( + "orders", + row["run_id"].as_str().expect("child id").to_owned(), + ) + }), + ) { + for kind in ["completed", execution.expected_selector] { + expected + .entry(format!("{source}_{kind}")) + .or_default() + .insert(id.clone()); + } + for filter in execution.expected_filters { + expected + .entry(format!("{source}_executed_{filter}")) + .or_default() + .insert(id.clone()); + } + } + } + } + let retry = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: source_ids[&( + "partial outcome is independent of selected-node success", + "daily_orders", + )], + from_failure: true, + reason: "Retry only failed orders without borrowing earlier customer success" + .into(), + priority: 10, + }; + let context = AuditContext { + actor: "test-admin".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + assert!( + request_rerun(store, &context, &retry) + .await + .expect("request rerun") + ); + assert!( + !request_rerun(store, &context, &retry) + .await + .expect("retry request") + ); + finish_claimed_run( + store, + &lease, + (location, "development"), + retry.run_id, + RunOutcome::Success, + [NodeOutcome::Success; 2], + [true; 2], + ) + .await; + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read rerun lineage"); + let (transaction, children) = query( + transaction, + "SELECT run_id FROM sqb_runs WHERE parent_run_id=$1", + vec![Parameter::Id(retry.run_id)], + ) + .await + .expect("rerun child"); + commit(transaction).await.expect("read rerun committed"); + for (source, id) in std::iter::once(("daily_orders", retry.run_id.to_string())).chain( + children.into_iter().map(|row| { + ( + "orders", + row["run_id"].as_str().expect("child id").to_owned(), + ) + }), + ) { + for suffix in ["completed", "success", "executed_orders_completed"] { + expected + .entry(format!("{source}_{suffix}")) + .or_default() + .insert(id.clone()); + } + } + let sources = list_runs(store, (location, "development"), 500) + .await + .expect("all source executions"); + // Same key in another target/location and a different job in this scope must not match. + let mut other_input = input.clone(); + other_input.location_id = other; + publish_snapshot(store, &other_input) + .await + .expect("other publication"); + other_input.location_id = location; + other_input.target = "preview".into(); + publish_snapshot(store, &other_input) + .await + .expect("other target"); + for scope in [(other, "development"), (location, "preview")] { + finish_run( + store, + &lease, + scope, + "orders", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + } + finish_run( + store, + &lease, + (location, "development"), + "inventory", + RunOutcome::Success, + [NodeOutcome::Success; 2], + [false; 2], + ) + .await; + // Recreate the exact pre-change index/migration state with populated run history. + let transaction = begin(store, TransactionMode::Write) + .await + .expect("upgrade fixture"); + let (transaction, _) = query(transaction, "DROP TABLE sqb_warehouse_freshness", vec![]) + .await + .expect("remove subsequent warehouse mirror migration"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_node_finished_cursor", vec![]) + .await + .expect("remove node finish cursor index"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_node_started_identity", vec![]) + .await + .expect("remove node start evidence index"); + let (transaction, _) = query(transaction, "DROP INDEX sqb_runs_parent", vec![]) + .await + .expect("remove execution lookup index"); + let (transaction, _) = + query(transaction, "DROP INDEX sqb_runs_terminal_cursor", vec![]) + .await + .expect("remove new index"); + let (transaction, _) = query( + transaction, + "DELETE FROM sqb_schema_migrations WHERE version>=6", + vec![], + ) + .await + .expect("remove new migration record"); + commit(transaction).await.expect("pre-change schema ready"); + migrate_store(store) + .await + .expect("upgrade populated prior schema"); + let replica = second_store(store).await; + let fault = TransactionFault { + description: "fail atomic sensor cursor", + table: "sqb_sensor_cursor_commits", + operation: "INSERT", + condition: "", + sqlite_body: "SELECT RAISE(ABORT, 'injected sensor failure')", + postgres_body: "RAISE EXCEPTION 'injected sensor failure'", + }; + let triggers = list_triggers(store, location, "development") + .await + .expect("triggers"); + for trigger in &triggers { + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + assert_eq!( + begin_sensor_evaluation(&replica, &lease, trigger.trigger_id) + .await + .expect("recover"), + Some(requested.clone()) + ); + install_fault(store, &fault).await; + assert!( + complete_builtin_sensor(store, &lease, &requested) + .await + .is_err() + ); + remove_fault(store, &fault).await; + let wanted = expected.get(&trigger.name).cloned().unwrap_or_default(); + assert_eq!( + complete_builtin_sensor(&replica, &lease, &requested) + .await + .expect("retry"), + wanted.len(), + "{}", + trigger.name + ); + assert_eq!( + complete_builtin_sensor(store, &lease, &requested) + .await + .expect("duplicate completion"), + 0 + ); + let transaction = begin(store, TransactionMode::Read) + .await + .expect("read requests"); + let (transaction, rows) = query( + transaction, + "SELECT run_key FROM sqb_run_requests WHERE trigger_id=$1", + vec![Parameter::Id(trigger.trigger_id)], + ) + .await + .expect("keys"); + commit(transaction).await.expect("read commit"); + let actual: BTreeSet = rows + .iter() + .map(|row| row["run_key"].as_str().expect("run key").into()) + .collect(); + assert_eq!(actual, wanted, "{}", trigger.name); + change(store, trigger.trigger_id, TriggerChange::ResetCursor).await; + } + tokio::time::sleep(Duration::from_secs(1)).await; + for trigger in &triggers { + let replay = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("reset request") + .expect("due"); + complete_builtin_sensor(store, &lease, &replay) + .await + .expect("replay deduplicates"); + } + let after = list_runs(store, (location, "development"), 500) + .await + .expect("queued sensors"); + assert_eq!( + after.len(), + sources.len() + 1 + test_case.expected_runs, + "{}: reset adds no runs", + test_case.description + ); + for source in sources { + assert_eq!( + after.iter().find(|record| record.run_id == source.run_id), + Some(&source), + "sensor never changes source state" + ); + } + close_store(replica).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_types.rs b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs new file mode 100644 index 0000000000..af27d0d35e --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_types.rs @@ -0,0 +1,24 @@ +use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::sensors::models::CustomSensorOutcome; +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct SensorTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_runs: usize, +} + +pub(crate) struct CustomSensorResultCase { + pub(crate) outcome: CustomSensorOutcome, + pub(crate) expected_phase: &'static str, + pub(crate) expected_cursor: Option<&'static str>, +} + +pub(crate) struct SensorExecutionTestCase { + pub(crate) description: &'static str, + pub(crate) outcome: RunOutcome, + pub(crate) nodes: [NodeOutcome; 2], + pub(crate) started: [bool; 2], + pub(crate) expected_selector: &'static str, + pub(crate) expected_filters: &'static [&'static str], +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs new file mode 100644 index 0000000000..2badd5ab9b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/helpers.rs @@ -0,0 +1,320 @@ +//! Real control-store integration coverage; both engines exercise the same public contract. + +use std::collections::BTreeMap; + +use crate::snapshots::main::current_snapshot::current_snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::{SnapshotInput, SnapshotNode}; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::tests::location_helpers::create_location; +use crate::snapshots::tests::test_types::SchemaRejectionCase; +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +fn snapshot(location_id: Uuid) -> SnapshotInput { + SnapshotInput { + location_id, + target: "development".into(), + generation: 1, + git_revision: None, + git_dirty: true, + git_author: None, + image_id: None, + environment_hash: "synthetic-environment".into(), + sqb_version: env!("CARGO_PKG_VERSION").into(), + nodes: BTreeMap::from([ + ( + "customers".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table", "columns": ["customer_id"]}), + raw_sql: Some("select 1 as customer_id".into()), + compiled_sql: Some("select 1 as customer_id".into()), + python_source: None, + }, + ), + ( + "orders".into(), + SnapshotNode { + kind: "model".into(), + definition: json!({"dependencies": ["customers"], "materialization": "view"}), + raw_sql: Some("select customer_id from ref('customers')".into()), + compiled_sql: Some("select customer_id from main.customers".into()), + python_source: Some("# display only: orders".into()), + }, + ), + ]), + documents: BTreeMap::from([("lineage".into(), json!({"orders": ["customers"]}))]), + } +} + +async fn exercise_snapshot_contract(store: &ControlStore) -> i64 { + let second = second_store(store).await; + let (first, second_migration) = tokio::join!(migrate_store(store), migrate_store(&second)); + first.expect("first migration succeeds"); + second_migration.expect("concurrent migration succeeds"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("create test location"); + assert!( + current_snapshot(store, location, "development") + .await + .expect("read empty target") + .is_none() + ); + let mut input = snapshot(location); + let initial = publish_snapshot(store, &input) + .await + .expect("publish initial snapshot"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read initial snapshot") + .expect("initial snapshot exists"); + assert_eq!(loaded.record, initial); + assert_eq!(loaded.nodes, input.nodes); + assert_eq!(loaded.documents, input.documents); + assert_eq!( + loaded + .changes + .iter() + .map(|change| (change.subject_key.as_str(), change.change_kind.as_str())) + .collect::>(), + [("customers", "added"), ("orders", "added")] + ); + + // Container insertion order and the generation do not affect the content address. + input.generation = 2; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .definition = json!({"materialization": "view", "dependencies": ["customers"]}); + let unchanged = publish_snapshot(store, &input) + .await + .expect("publish reordered definition"); + assert_eq!(unchanged.manifest_digest, initial.manifest_digest); + assert!( + current_snapshot(store, location, "development") + .await + .expect("read unchanged snapshot") + .expect("unchanged snapshot exists") + .changes + .is_empty() + ); + assert!(matches!( + publish_snapshot(store, &input).await, + Err(ControlError::StaleGeneration) + )); + + input.generation = 3; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .compiled_sql = Some("select customer_id + 5 from main.customers".into()); + let changed = publish_snapshot(store, &input) + .await + .expect("publish changed SQL"); + assert_ne!(changed.manifest_digest, unchanged.manifest_digest); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read changed snapshot") + .expect("changed snapshot exists"); + assert_eq!(loaded.nodes, input.nodes); + assert_eq!(loaded.changes.len(), 1); + assert_eq!(loaded.changes[0].subject_key, "orders"); + assert_eq!(loaded.changes[0].change_kind, "changed"); + assert_ne!(loaded.changes[0].before_hash, loaded.changes[0].after_hash); + + // One target's generation does not suppress another target's first publication. + input.target = "testing".into(); + input.generation = 1; + publish_snapshot(store, &input) + .await + .expect("publish another target"); + assert_eq!( + current_snapshot(store, location, "development") + .await + .expect("read development target") + .expect("development target exists") + .record + .generation, + 3 + ); + assert_eq!( + current_snapshot(store, location, "testing") + .await + .expect("read testing target") + .expect("testing target exists") + .record + .generation, + 1 + ); + + input.target = "development".into(); + input.generation = 4; + input.nodes.remove("orders"); + publish_snapshot(store, &input) + .await + .expect("publish node removal"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read node removal") + .expect("snapshot exists after removal"); + assert_eq!(loaded.changes.len(), 1); + assert_eq!(loaded.changes[0].change_kind, "removed"); + assert_eq!(loaded.changes[0].subject_key, "orders"); + assert!(loaded.changes[0].after_hash.is_none()); + + // The newer compiler wins regardless of which publisher takes the transaction lock first. + let mut older = input.clone(); + older.generation = 5; + input.generation = 6; + input + .nodes + .get_mut("customers") + .expect("customers node exists") + .compiled_sql = Some("select 6 as customer_id".into()); + let (old_result, new_result) = tokio::join!( + publish_snapshot(store, &older), + publish_snapshot(&second, &input) + ); + assert!(old_result.is_ok() || matches!(old_result, Err(ControlError::StaleGeneration))); + new_result.expect("new generation wins"); + let loaded = current_snapshot(store, location, "development") + .await + .expect("read winning snapshot") + .expect("winning snapshot exists"); + assert_eq!(loaded.record.generation, 6); + assert_eq!(loaded.nodes, input.nodes); + close_store(second).await; + loaded.record.generation +} + +pub(super) async fn snapshot_contract(backend: Backend) -> Option { + let test_store = open_test_store(backend).await?; + let generation = exercise_snapshot_contract(&test_store.store).await; + verify_sqlite_deduplication(&test_store.store); + close_test_store(test_store).await; + Some(generation) +} + +fn verify_sqlite_deduplication(store: &ControlStore) { + let ControlStore::Sqlite(path) = store else { + return; + }; + let connection = rusqlite::Connection::open(path).expect("inspect control store"); + let definitions: i64 = connection + .query_row("SELECT count(*) FROM sqb_node_definitions", [], |row| { + row.get(0) + }) + .expect("count immutable definitions"); + // Two initial nodes, one changed source, and one changed customer; unchanged targets reuse them. + assert_eq!(definitions, 4); + let sources: i64 = connection + .query_row("SELECT count(*) FROM sqb_source_texts", [], |row| { + row.get(0) + }) + .expect("count shared sources"); + assert_eq!(sources, 6); +} + +pub(super) async fn sqlite_failed_publish_and_schema_floor() -> i64 { + let directory = tempfile::tempdir().expect("create test directory"); + let path = directory.path().join("control.sqlite3"); + let store = open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open test store"); + migrate_store(&store).await.expect("migrate test store"); + let location = create_location(&store, "orders", "test-admin") + .await + .expect("create test location"); + let mut input = snapshot(location); + let good = publish_snapshot(&store, &input) + .await + .expect("publish good snapshot"); + let connection = rusqlite::Connection::open(&path).expect("open fault injection connection"); + connection + .execute_batch( + "DROP TABLE sqb_warehouse_freshness; \ + DROP INDEX sqb_node_finished_cursor; DROP INDEX sqb_node_started_identity; \ + DROP INDEX sqb_runs_parent; DROP INDEX sqb_runs_terminal_cursor; \ + DROP TABLE sqb_run_requests; DROP TABLE sqb_sensor_cursor_commits; \ + DROP TABLE sqb_sensor_evaluation_events; DROP TABLE sqb_platform_sink_deliveries; \ + DROP TABLE sqb_location_work_results; DROP TABLE sqb_location_work; \ + DROP TABLE sqb_location_instances; DELETE FROM sqb_schema_migrations WHERE version>=2", + ) + .expect("reconstruct the previous schema without changing existing facts"); + migrate_store(&store) + .await + .expect("upgrade the previous schema"); + migrate_store(&store) + .await + .expect("repeat the migration without changes"); + assert_eq!( + current_snapshot(&store, location, "development") + .await + .expect("read upgraded store") + .expect("snapshot survives upgrade") + .record, + good + ); + // Fail after immutable bodies were inserted but before switching the current pointer. + connection.execute_batch("CREATE TRIGGER fail_snapshot_switch BEFORE UPDATE ON sqb_current_snapshots BEGIN SELECT RAISE(ABORT, 'synthetic publication failure'); END").expect("install publication failure"); + input.generation = 2; + input + .nodes + .get_mut("orders") + .expect("orders node exists") + .compiled_sql = Some("select 99".into()); + assert!(publish_snapshot(&store, &input).await.is_err()); + assert_eq!( + current_snapshot(&store, location, "development") + .await + .expect("read preserved snapshot") + .expect("last good snapshot exists") + .record, + good + ); + let count: i64 = connection + .query_row("SELECT count(*) FROM sqb_snapshots", [], |row| row.get(0)) + .expect("count committed snapshots"); + assert_eq!(count, 1); + connection + .execute_batch("DROP TRIGGER fail_snapshot_switch") + .expect("remove injected failure"); + let recovered = publish_snapshot(&store, &input) + .await + .expect("retry publication"); + for case in [ + SchemaRejectionCase { + description: "minimum server version is newer", + update_sql: "UPDATE sqb_schema_migrations SET min_server_version='9999.0.0'", + }, + SchemaRejectionCase { + description: "migration checksum changed", + update_sql: "UPDATE sqb_schema_migrations SET min_server_version='0.0.0', checksum='different'", + }, + SchemaRejectionCase { + description: "migration version is unknown", + update_sql: "UPDATE sqb_schema_migrations SET version=999 WHERE version=2", + }, + ] { + connection + .execute_batch(case.update_sql) + .expect("apply synthetic schema incompatibility"); + assert!( + matches!(migrate_store(&store).await, Err(ControlError::Schema(_))), + "{}", + case.description + ); + } + recovered.generation +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs new file mode 100644 index 0000000000..708a6aff32 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs @@ -0,0 +1,48 @@ +use crate::snapshots::tests::helpers::{snapshot_contract, sqlite_failed_publish_and_schema_floor}; +use crate::snapshots::tests::test_types::{RecoveryTestCase, SnapshotTestCase}; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_control_store_when_snapshots_publish_then_generations_sources_and_changes_are_consistent() +{ + let test_cases = [ + SnapshotTestCase { + description: "SQLite snapshot publication", + backend: Backend::Sqlite, + expected_generation: 6, + }, + SnapshotTestCase { + description: "PostgreSQL snapshot publication", + backend: Backend::Postgres, + expected_generation: 6, + }, + ]; + for test_case in test_cases { + let Some(observed) = run(snapshot_contract(test_case.backend)) else { + continue; + }; + assert_eq!( + observed, test_case.expected_generation, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sqlite_when_a_publish_fails_then_last_good_snapshot_survives_and_schema_drift_is_rejected() +{ + let test_cases = [RecoveryTestCase { + description: "atomic rollback and recovery", + expected_generation: 2, + }]; + for test_case in test_cases { + let observed = run(sqlite_failed_publish_and_schema_floor()); + assert_eq!( + observed, test_case.expected_generation, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs new file mode 100644 index 0000000000..6963cfddf6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_contracts.rs @@ -0,0 +1,8 @@ +//! Snapshot contracts compiled within the snapshot owner. + +#[path = "test_contract.rs"] +mod contract; +#[path = "helpers.rs"] +mod helpers; +#[path = "test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs new file mode 100644 index 0000000000..61a2c6832b --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/snapshots/test_types.rs @@ -0,0 +1,22 @@ +//! Cases that alter a disposable schema to verify startup rejection. + +use crate::store::tests::store_test_types::Backend; + +#[derive(Debug)] +pub(super) struct SchemaRejectionCase { + pub(super) description: &'static str, + pub(super) update_sql: &'static str, +} + +#[derive(Debug)] +pub(super) struct SnapshotTestCase { + pub(super) description: &'static str, + pub(super) backend: Backend, + pub(super) expected_generation: i64, +} + +#[derive(Debug)] +pub(super) struct RecoveryTestCase { + pub(super) description: &'static str, + pub(super) expected_generation: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs new file mode 100644 index 0000000000..8c221edcc0 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/faults/helpers.rs @@ -0,0 +1,47 @@ +//! Database-trigger fault injection at durable publication boundaries. + +use crate::store::models::ControlStore; + +use crate::store::tests::fault_types::TransactionFault; + +pub(crate) async fn install_fault(store: &ControlStore, point: &TransactionFault) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch(&format!( + "CREATE TRIGGER sqb_test_fault BEFORE {} ON {} {} BEGIN {}; END;", + point.operation, point.table, point.condition, point.sqlite_body + )) + .expect(point.description); + return; + }; + sqlx_core::query::query(&format!("CREATE FUNCTION sqb_test_fault() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN {}; RETURN NEW; END $$", point.postgres_body)) + .execute(pool).await.expect("install PostgreSQL fault function"); + sqlx_core::query::query(&format!("CREATE TRIGGER sqb_test_fault BEFORE {} ON {} FOR EACH ROW {} EXECUTE FUNCTION sqb_test_fault()", + point.operation, point.table, point.condition.replace("WHEN ", "WHEN (") + [")", ""][usize::from(point.condition.is_empty())])) + .execute(pool).await.expect(point.description); +} + +pub(crate) async fn remove_fault(store: &ControlStore, point: &TransactionFault) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("fault connection") + .execute_batch("DROP TRIGGER sqb_test_fault") + .expect("remove SQLite transaction fault"); + return; + }; + sqlx_core::query::query(&format!("DROP TRIGGER sqb_test_fault ON {}", point.table)) + .execute(pool) + .await + .expect("remove PostgreSQL fault trigger"); + sqlx_core::query::query("DROP FUNCTION sqb_test_fault()") + .execute(pool) + .await + .expect("remove PostgreSQL fault function"); +} diff --git a/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs b/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs new file mode 100644 index 0000000000..e25282c3cc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/faults/test_types.rs @@ -0,0 +1,10 @@ +//! Transaction failure cases shared by credential, tick and run contracts. + +pub(crate) struct TransactionFault { + pub(crate) description: &'static str, + pub(crate) table: &'static str, + pub(crate) operation: &'static str, + pub(crate) condition: &'static str, + pub(crate) sqlite_body: &'static str, + pub(crate) postgres_body: &'static str, +} diff --git a/crates/sqlbuild-orchestration/tests/store/helpers.rs b/crates/sqlbuild-orchestration/tests/store/helpers.rs new file mode 100644 index 0000000000..ba5fd692cc --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/helpers.rs @@ -0,0 +1,75 @@ +//! Real-database setup shared by storage, scheduling, and queue contracts. + +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; +use uuid::Uuid; + +use crate::store::tests::store_test_types::{Backend, TestStore}; + +pub(crate) fn run(future: impl std::future::Future) -> T { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create control-store test runtime") + .block_on(future) +} + +pub(crate) async fn open_test_store(backend: Backend) -> Option { + let Backend::Postgres = backend else { + let directory = tempfile::tempdir().expect("create isolated SQLite directory"); + let store = open_store( + directory + .path() + .join("control.sqlite3") + .to_str() + .expect("UTF-8 test path"), + ) + .await + .expect("open isolated SQLite store"); + return Some(TestStore { + store, + directory: Some(directory), + postgres: None, + }); + }; + let url = std::env::var("SQLBUILD_TEST_CONTROL_POSTGRES_URL").expect( + "PostgreSQL integration requires the disposable database harness; run make test-native", + ); + let admin = sqlx_postgres::PgPool::connect(&url) + .await + .expect("connect to test PostgreSQL"); + let schema = format!("sqb_test_{}", Uuid::now_v7().simple()); + sqlx_core::query::query(&format!("CREATE SCHEMA {schema}")) + .execute(&admin) + .await + .expect("create isolated test schema"); + let separator = ['?', '&'][usize::from(url.contains('?'))]; + let store = open_store(&format!("{url}{separator}options=-csearch_path%3D{schema}")) + .await + .expect("open isolated PostgreSQL store"); + Some(TestStore { + store, + directory: None, + postgres: Some((admin, schema)), + }) +} + +pub(crate) async fn close_test_store(test_store: TestStore) { + close_store(test_store.store).await; + let Some((admin, schema)) = test_store.postgres else { + return; + }; + sqlx_core::query::query(&format!("DROP SCHEMA {schema} CASCADE")) + .execute(&admin) + .await + .expect("remove isolated test schema"); + admin.close().await; + drop(test_store.directory); +} + +pub(crate) async fn close_store(store: ControlStore) { + let ControlStore::Postgres(pool) = store else { + return; + }; + pool.close().await; +} diff --git a/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs new file mode 100644 index 0000000000..59f067dc23 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/leases/helpers.rs @@ -0,0 +1,24 @@ +//! Expire the persisted lease without introducing a fake control-store clock. + +use crate::store::models::ControlStore; + +pub(crate) async fn expire_lease(store: &ControlStore) { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + rusqlite::Connection::open(path) + .expect("open lease expiry connection") + .execute_batch( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .expect("expire SQLite scheduler lease"); + return; + }; + sqlx_core::query::query( + "UPDATE sqb_scheduler_lease SET expires_at='2000-01-01T00:00:00.000000Z'", + ) + .execute(pool) + .await + .expect("expire PostgreSQL scheduler lease"); +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs new file mode 100644 index 0000000000..a530b60e5c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/recall/helpers.rs @@ -0,0 +1,179 @@ +//! Recall publication fixtures exercise the control store's durable append boundary. + +use crate::store::errors::ControlError; +use crate::store::main::append_recall::append_recall; +use crate::store::main::begin::begin; +use crate::store::main::commit::commit; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter, RecallAppendOutcome, TransactionMode}; +use crate::store::types::Row; +use serde_json::json; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::models::{ArtifactInput, RecallRow, RowEnvelope}; + +use crate::store::tests::replica_helpers::second_store; +use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store}; +use crate::store::tests::store_test_types::Backend; + +pub(crate) async fn publish_recall( + store: &ControlStore, + rows: &[RecallRow], +) -> Result, ControlError> { + let mut transaction = begin(store, TransactionMode::Write).await?; + let mut outcomes = Vec::with_capacity(rows.len()); + for row in rows { + let outcome; + (transaction, outcome) = append_recall(transaction, row).await?; + outcomes.push(outcome); + } + commit(transaction).await?; + Ok(outcomes) +} + +pub(crate) async fn read_recall_events( + store: &ControlStore, + after: i64, + limit: u32, +) -> Result, ControlError> { + let transaction = begin(store, TransactionMode::Read).await?; + let (transaction, rows) = query( + transaction, + "SELECT * FROM recall_events WHERE storage_id > $1 ORDER BY storage_id LIMIT $2", + vec![ + Parameter::Integer(after), + Parameter::Integer(i64::from(limit.min(1000))), + ], + ) + .await?; + commit(transaction).await?; + Ok(rows) +} + +pub(crate) fn event(id: &str, occurred_at: &str, duration_ms: f64) -> RecallRow { + event_object_row(&RowEnvelope { invocation_id: "orders-build".into(), + invocation_started_at: "2026-03-01T00:30:00+01:00".into(), producer_version: "test".into() }, + json!({ "event_id": id, "event_type": "invocation_completed", "schema_version": 1, + "occurred_at": occurred_at, "producer": "worker", "invocation_sequence": 1, + "payload": { "duration_ms": duration_ms, "exit_code": 0, "nested": [null, true, "orders"] } }) + .as_object().expect("event object")).expect("valid lifecycle event") +} + +pub(crate) async fn recall_contract(backend: Backend) -> Option> { + let fixture = open_test_store(backend).await?; + let store = &fixture.store; + migrate_store(store) + .await + .expect("migrate control and Recall schema"); + let second = second_store(store).await; + let first = event("orders-first", "2026-03-01T00:00:00Z", 3.25); + let first_batch = [first.clone()]; + let (left, right) = tokio::join!( + publish_recall(store, &first_batch), + publish_recall(&second, &first_batch) + ); + let left = left.expect("first writer").remove(0); + let right = right.expect("second writer").remove(0); + assert_ne!(left.inserted, right.inserted, "exactly one writer inserts"); + assert_eq!( + left.storage_id, right.storage_id, + "duplicates preserve cursor" + ); + let cursor = left.storage_id.expect("event storage cursor"); + let conflict = publish_recall(store, &[event("orders-first", "2026-03-01T00:00:00Z", 7.5)]) + .await + .expect("record conflicting retry") + .remove(0); + assert!( + conflict.conflict.is_some(), + "different content records conflict" + ); + let later = event("orders-later-arrival", "2026-02-01T00:00:00Z", 2.5); + let malformed = event("orders-malformed", "2026-04-01T00:00:00Z", 1.0); + let mut malformed = malformed; + malformed.values.retain(|(name, _)| *name != "event_id"); + assert!( + publish_recall(store, &[later.clone(), malformed]) + .await + .is_err(), + "whole malformed batch rolls back" + ); + assert!( + read_recall_events(store, cursor, 100) + .await + .expect("rolled-back page") + .is_empty() + ); + publish_recall(store, &[later]) + .await + .expect("retry rolled-back event"); + let page = read_recall_events(store, cursor, 1) + .await + .expect("late arrival page"); + assert_eq!(page.len(), 1); + assert_eq!( + page[0]["event_id"], "orders-later-arrival", + "page follows commit order, not producer time" + ); + let rows = read_recall_events(store, 0, 100).await.expect("all events"); + assert_eq!( + rows[0]["duration_ms"], + json!(3.25), + "first content wins with fractional precision" + ); + let payload = rows[0]["payload"].as_str().map_or_else( + || rows[0]["payload"].clone(), + |text| serde_json::from_str(text).expect("JSON payload"), + ); + assert_eq!(payload["nested"], json!([null, true, "orders"])); + let artifact = ArtifactInput { + digest: "sha256:orders".into(), + kind: "sql".into(), + media_type: "application/octet-stream".into(), + content: vec![0, 255, 1, 0], + seen_at: "2026-01-01T00:00:00Z".into(), + ref_id: "orders".into(), + role: "input".into(), + statement_id: None, + resource_id: None, + }; + let first_artifact = artifact_rows( + &RowEnvelope { + invocation_id: "first-artifact".into(), + invocation_started_at: artifact.seen_at.clone(), + producer_version: "test".into(), + }, + &artifact, + ); + publish_recall(store, &first_artifact) + .await + .expect("first binary artifact"); + let artifact = ArtifactInput { + seen_at: "2026-04-01T00:00:00Z".into(), + ..artifact + }; + let second_artifact = artifact_rows( + &RowEnvelope { + invocation_id: "second-artifact".into(), + invocation_started_at: artifact.seen_at.clone(), + producer_version: "test".into(), + }, + &artifact, + ); + let outcomes = publish_recall(store, &second_artifact) + .await + .expect("same artifact across months"); + assert!( + !outcomes[0].inserted && outcomes[0].conflict.is_none(), + "artifact identity is global, not partition-local" + ); + assert!(outcomes[1].inserted, "each invocation keeps its reference"); + let ids = rows + .iter() + .map(|row| row["event_id"].as_str().expect("event ID").to_owned()) + .collect(); + close_store(second).await; + close_test_store(fixture).await; + Some(ids) +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs new file mode 100644 index 0000000000..988e9542be --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_contract.rs @@ -0,0 +1,30 @@ +//! Idempotency and commit-order paging through real store transactions. + +use crate::store::tests::helpers::recall_contract; +use crate::store::tests::store_helpers::run; +use crate::store::tests::store_test_types::Backend; +use crate::store::tests::test_types::RecallTestCase; + +#[test] +fn given_native_recall_facts_when_retried_and_paged_then_content_and_commit_order_are_preserved() { + let test_cases = [ + RecallTestCase { + description: "SQLite Recall transactions", + backend: Backend::Sqlite, + expected_event_ids: vec!["orders-first", "orders-later-arrival"], + }, + RecallTestCase { + description: "PostgreSQL Recall transactions", + backend: Backend::Postgres, + expected_event_ids: vec!["orders-first", "orders-later-arrival"], + }, + ]; + for test_case in test_cases { + let observed = run(recall_contract(test_case.backend)); + assert!( + observed.is_none_or(|ids| ids == test_case.expected_event_ids), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs new file mode 100644 index 0000000000..53cbf105a6 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_types.rs @@ -0,0 +1,15 @@ +//! Expected Recall storage outcomes for both supported databases. + +use crate::store::tests::store_test_types::Backend; + +pub(crate) struct RecallTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_event_ids: Vec<&'static str>, +} + +pub(crate) struct RecallViewTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_complete: i64, +} diff --git a/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs new file mode 100644 index 0000000000..a157ecf787 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/recall/test_views.rs @@ -0,0 +1,194 @@ +//! Durable Recall views preserve the published execution records. + +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::read_recall_view::read_recall_view; +use serde_json::json; +use sqlbuild_recall::records::main::artifact_rows::artifact_rows; +use sqlbuild_recall::records::main::delivery_row::delivery_row; +use sqlbuild_recall::records::main::event_object_row::event_object_row; +use sqlbuild_recall::records::main::heartbeat_row::heartbeat_row; +use sqlbuild_recall::records::main::invocation_row::invocation_row; +use sqlbuild_recall::records::main::output_row::output_row; +use sqlbuild_recall::records::models::{ + ArtifactInput, DeliveryCounts, InvocationContext, OutputChunk, OutputStream, RowEnvelope, +}; + +use crate::store::tests::helpers::publish_recall; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::store::tests::test_types::RecallViewTestCase; + +#[test] +fn given_native_context_output_and_audits_when_reading_views_then_both_databases_preserve_their_meaning() + { + let test_cases = [ + RecallViewTestCase { + description: "SQLite supported views", + backend: Backend::Sqlite, + expected_complete: 1, + }, + RecallViewTestCase { + description: "PostgreSQL supported views", + backend: Backend::Postgres, + expected_complete: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate view store"); + let envelope = RowEnvelope { + invocation_id: "orders-output".into(), + invocation_started_at: "2026-01-02T00:00:00Z".into(), + producer_version: "test".into(), + }; + let context = InvocationContext { + command: Some("build".into()), + selection: vec!["orders".into()], + external_context: Some( + json!({"integration": {"name": "orders", "retry_number": 2}}).to_string(), + ), + ..InvocationContext::default() + }; + let mut rows = vec![ + invocation_row(&envelope, &context), + heartbeat_row( + &envelope, + ("2026-01-02T00:00:01Z", "2026-01-02T00:00:31Z"), + (None, None), + (Some("build"), None), + ), + ]; + for (sequence, (stream, bytes)) in [ + (OutputStream::Stdout, b"orders\x00\xff".as_slice()), + (OutputStream::Stderr, b"warning\n".as_slice()), + (OutputStream::Stdout, b"\xc3\x28 tail".as_slice()), + ] + .into_iter() + .enumerate() + { + rows.push(output_row( + &envelope, + &OutputChunk { + sequence: sequence as u64, + stream, + occurred_at: "2026-01-02T00:00:02Z".into(), + data: bytes.to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + }, + )); + } + rows.push(delivery_row( + &envelope, + "output", + ( + &DeliveryCounts { + accepted: 3, + delivered: 3, + ..DeliveryCounts::default() + }, + true, + ), + "2026-01-02T00:00:03Z", + )); + for (event_id, event_type, payload) in [ + ( + "orders-audit", + "audit_completed", + json!({"audit_name":"unique_orders", "violation_count": 7, "measured_value": 3.5, "target_name":"orders", "evidence_truncated": true}), + ), + ( + "orders-complete", + "invocation_completed", + json!({"duration_ms": 12.75, "exit_code": 0}), + ), + ] { + rows.push(event_object_row(&envelope, json!({"event_id": event_id, "event_type": event_type, "schema_version": 1, "occurred_at": "2026-01-02T00:00:03Z", "producer": "worker", "invocation_sequence": 1, "payload": payload}).as_object().expect("event object")).expect("lifecycle fact")); + } + rows.extend(artifact_rows( + &envelope, + &ArtifactInput { + digest: "sha256:orders-display".into(), + kind: "sql".into(), + media_type: "text/plain".into(), + content: b"orders\x00\xff".to_vec(), + seen_at: envelope.invocation_started_at.clone(), + ref_id: "orders-statement".into(), + role: "input".into(), + statement_id: None, + resource_id: None, + }, + )); + publish_recall(store, &rows) + .await + .expect("write view inputs"); + let output = + read_recall_view(store, ("output_invocations", &envelope.invocation_id), 100) + .await + .expect("output accounting"); + assert_eq!(output[0]["complete"], json!(test_case.expected_complete)); + assert_eq!(output[0]["chunks"], json!(3)); + assert_eq!(output[0]["missing_sequences"], json!(0)); + let transcript = + read_recall_view(store, ("output_transcripts", &envelope.invocation_id), 100) + .await + .expect("binary transcript"); + assert_eq!( + transcript[0]["stdout"], + json!([b"orders\x00\xff".as_slice(), b"\xc3\x28 tail".as_slice()].concat()) + ); + assert_eq!( + transcript[0]["combined"], + json!( + [ + b"orders\x00\xff".as_slice(), + b"warning\n".as_slice(), + b"\xc3\x28 tail".as_slice() + ] + .concat() + ) + ); + let invocation = read_recall_view(store, ("invocations", &envelope.invocation_id), 100) + .await + .expect("invocation projection"); + assert_eq!(invocation[0]["status"], "completed"); + assert_eq!(invocation[0]["duration_ms"], json!(12.75)); + assert_eq!(invocation[0]["integration_retry"], json!(2)); + let audit = read_recall_view(store, ("audit_results", &envelope.invocation_id), 100) + .await + .expect("audit projection"); + assert_eq!(audit[0]["violation_count"], json!(7)); + assert_eq!(audit[0]["measured_value"], json!(3.5)); + assert_eq!(audit[0]["evidence_truncated"], json!(1)); + let tail = read_recall_view(store, ("output_tail", &envelope.invocation_id), 100) + .await + .expect("lossy output display"); + assert_eq!( + tail.iter() + .find(|row| row["sequence"] == 0) + .expect("first chunk")["text"], + json!("orders\u{0}\u{fffd}"), + "{}", + test_case.description + ); + assert_eq!( + tail.iter() + .find(|row| row["sequence"] == 2) + .expect("last chunk")["text"], + json!("\u{fffd}( tail") + ); + let artifacts = read_recall_view(store, ("artifacts", &envelope.invocation_id), 100) + .await + .expect("lossy artifact display"); + assert_eq!(artifacts[0]["text"], json!("orders\u{0}\u{fffd}")); + assert_eq!(artifacts[0]["content"], json!(b"orders\x00\xff".to_vec())); + assert_eq!(artifacts[0]["available"], json!(1)); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs new file mode 100644 index 0000000000..443b0e5e26 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/replicas/helpers.rs @@ -0,0 +1,20 @@ +//! Independent database handles for multi-replica integration contracts. + +use crate::store::main::open_store::open_store; +use crate::store::models::ControlStore; + +pub(crate) async fn second_store(store: &ControlStore) -> ControlStore { + let ControlStore::Postgres(pool) = store else { + let ControlStore::Sqlite(path) = store else { + unreachable!() + }; + return open_store(path.to_str().expect("UTF-8 test path")) + .await + .expect("open independent SQLite handle"); + }; + ControlStore::Postgres( + sqlx_postgres::PgPool::connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("open independent PostgreSQL pool"), + ) +} diff --git a/crates/sqlbuild-orchestration/tests/store/test_contracts.rs b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs new file mode 100644 index 0000000000..a257c9aeb9 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/test_contracts.rs @@ -0,0 +1,22 @@ +//! Recall storage integration contracts for SQLite and PostgreSQL. + +#[path = "faults/helpers.rs"] +pub(crate) mod fault_helpers; +#[path = "faults/test_types.rs"] +pub(crate) mod fault_types; +#[path = "recall/helpers.rs"] +pub(crate) mod helpers; +#[path = "leases/helpers.rs"] +pub(crate) mod lease_helpers; +#[path = "replicas/helpers.rs"] +pub(crate) mod replica_helpers; +#[path = "helpers.rs"] +pub(crate) mod store_helpers; +#[path = "test_types.rs"] +pub(crate) mod store_test_types; +#[path = "recall/test_contract.rs"] +mod test_contract; +#[path = "recall/test_types.rs"] +mod test_types; +#[path = "recall/test_views.rs"] +mod test_views; diff --git a/crates/sqlbuild-orchestration/tests/store/test_types.rs b/crates/sqlbuild-orchestration/tests/store/test_types.rs new file mode 100644 index 0000000000..1ffddfd679 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/store/test_types.rs @@ -0,0 +1,16 @@ +//! Store fixtures retain their isolated SQLite directory or PostgreSQL schema until cleanup. + +use crate::store::models::ControlStore; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Backend { + Sqlite, + Postgres, +} + +#[derive(Debug)] +pub(crate) struct TestStore { + pub(crate) store: ControlStore, + pub(crate) directory: Option, + pub(crate) postgres: Option<(sqlx_postgres::PgPool, String)>, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 661bd3d6a1..621a30947c 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -16,6 +16,8 @@ default = [] extension-module = ["pyo3/extension-module"] [dependencies] +chrono.workspace = true +chrono-tz.workspace = true fensu-policy.workspace = true mimalloc.workspace = true pyo3.workspace = true @@ -24,6 +26,7 @@ rusqlite.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true signal-hook.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true @@ -35,6 +38,7 @@ sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-host.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true sqlbuild-project.workspace = true sqlbuild-pysource.workspace = true sqlbuild-recall.workspace = true @@ -45,6 +49,8 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true stacker.workspace = true +tokio.workspace = true +uuid.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs index 5f2f304c6b..0a444023a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs @@ -104,3 +104,35 @@ pub(crate) fn config_value_to_python(py: Python<'_>, value: ConfigValue) -> PyRe } } } + +/// JSON-backed configuration keeps integer widths and recursively constructs native Python values. +pub(crate) fn json_value_to_python<'py>( + py: Python<'py>, + value: &serde_json::Value, +) -> PyResult> { + use serde_json::Value; + match value { + Value::Null => Ok(py.None().into_bound(py)), + Value::Bool(value) => value.into_bound_py_any(py), + Value::String(value) => value.into_bound_py_any(py), + Value::Number(value) => match (value.as_i64(), value.as_u64()) { + (Some(value), _) => value.into_bound_py_any(py), + (None, Some(value)) => value.into_bound_py_any(py), + (None, None) => value.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py), + }, + Value::Array(items) => { + let values = items + .iter() + .map(|item| json_value_to_python(py, item)) + .collect::>>()?; + Ok(PyList::new(py, values)?.into_any()) + } + Value::Object(entries) => { + let table = PyDict::new(py); + for (key, item) in entries { + table.set_item(key, json_value_to_python(py, item)?)?; + } + Ok(table.into_any()) + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 8dbd45566b..3f45271352 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -1,5 +1,6 @@ //! `sqb build`: resolve, compile, gate, execute and report one direct build. +use std::sync::Arc; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -17,6 +18,7 @@ use crate::bindings::_helpers::build_command::planning::{ compile_build_plan, enforce_build_plan_policies, run_defer_clone_prephase, write_compile_target, write_plan_text, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; @@ -38,6 +40,14 @@ fn run_with_providers(context: &BuildContext<'_>) -> PyResult { } fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { + if let Some(worker) = &context.worker { + worker.check_cancelled(context.py())?; + if !context.get("effective_defer_clone_from")?.is_none() { + return Err(pyo3::exceptions::PyValueError::new_err( + "worker defer-clone requires claimed boundary writers and is not yet supported", + )); + } + } run_defer_clone_prephase(context)?; let cache_scope: Bound<'_, PyAny> = call_with( &host::entry(context.py(), AdapterHost::RelationMetadataCacheScope)?, @@ -49,6 +59,9 @@ fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) - /// Plan and run the build, sharing one relation metadata cache from planning to the run's end. fn run_cached_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; + if let Some(worker) = &context.worker { + worker.enforce_plan_writers(&pipeline)?; + } write_plan_text(context, &pipeline)?; enforce_build_plan_policies(context, &pipeline)?; write_compile_target(context, &pipeline)?; @@ -74,9 +87,11 @@ fn run_within_limits( request: &Bound<'_, PyAny>, tracker: &Bound<'_, PyAny>, started: Instant, + worker: Option>, ) -> PyResult { let py: Python<'_> = request.py(); - let context: BuildContext<'_> = resolve_build_context(request, tracker, started)?; + let mut context: BuildContext<'_> = resolve_build_context(request, tracker, started)?; + context.worker = worker; let limits: Bound<'_, PyAny> = context.get("execution_limits")?; let deadline: Bound<'_, PyAny> = call_with( &host::entry(py, ExecutorHost::ExecutionDeadlineScope)?, @@ -97,6 +112,15 @@ fn run_within_limits( #[pyfunction] #[pyo3(signature = (*, request))] pub(crate) fn run_build_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + run_build(py, request, None) +} + +/// A claim owner survives planning, execution and connection cleanup before terminal reporting. +pub(crate) fn run_build( + py: Python<'_>, + request: &Bound<'_, PyAny>, + worker: Option>, +) -> PyResult { let tracker: Bound<'_, PyAny> = BuildPhaseTimingTracker::new(py, None) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; @@ -107,7 +131,7 @@ pub(crate) fn run_build_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> P within(&reporting, |_| { within(&tracker.call_method0("scope")?, |_| { let started: Instant = Instant::now(); - run_within_limits(request, &tracker, started).inspect_err(|_| { + run_within_limits(request, &tracker, started, worker.clone()).inspect_err(|_| { let verbose: bool = native_field(request, "verbose") .and_then(|value| value.is_truthy()) .unwrap_or(false) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index f0a0f55d12..41e53c84da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -225,6 +225,10 @@ pub(crate) fn prepare_build_execution<'py>( ], )?), ) + .map(|mut value| { + value.worker = context.worker.clone(); + value + }) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; let concurrency = context.concurrency()?.limit; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 1f1b394cb0..a2a5d45df6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -1,5 +1,6 @@ //! Resolve discovery, target, adapter, limits, connection and reporters for one build. +use std::sync::Arc; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -19,6 +20,7 @@ use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; @@ -38,6 +40,7 @@ pub(crate) struct BuildContext<'py> { pub(crate) verbose: bool, pub(crate) started: Instant, pub(crate) tracker: Bound<'py, PyAny>, + pub(crate) worker: Option>, } impl<'py> BuildContext<'py> { @@ -241,5 +244,6 @@ pub(crate) fn resolve_build_context<'py>( verbose: native_field(request, "verbose")?.is_truthy()? || debug, started, tracker: tracker.clone(), + worker: None, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs index ec6bbad153..62e6f13740 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -1,7 +1,7 @@ //! Live progress rows, completion rows and runtime diagnostics of `sqb build`. use std::collections::HashMap; -use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Instant; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -31,6 +31,7 @@ use crate::bindings::_helpers::build_progress::terminal::{ consume_terminal, flush, join_spinner, projector, release, shared_lines, start_spinner, terminal_columns, tty_flag, write, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -85,11 +86,8 @@ fn sql_test_resource_id(result: &Bound<'_, PyAny>) -> PyResult { } /// The resource name and projector identity of a completion. -fn resource_identity( - result: &Bound<'_, PyAny>, - kind: ResultKind, -) -> PyResult> { - Ok(match kind { +pub(crate) fn resource_identity(result: &Bound<'_, PyAny>) -> PyResult> { + Ok(match result_kind(result)? { ResultKind::Model => { let name: String = text(result, "model_name")?; Some((name.clone(), format!("model:{name}"))) @@ -141,6 +139,7 @@ struct CallbackState { /// Encapsulates live build progress output state and callbacks. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct BuildProgressCallbacks { + pub(crate) worker: Option>, plan: Py, stream: Py, lines: Py, @@ -453,6 +452,7 @@ impl BuildProgressCallbacks { } let lines: Bound<'_, PyAny> = shared_lines(py)?; Ok(Self { + worker: None, plan: plan.unbind(), is_tty: tty_flag(&stdout)? && !debug, stream: stream.unbind(), @@ -484,6 +484,9 @@ impl BuildProgressCallbacks { ) -> PyResult<()> { let py: Python<'_> = slf.py(); let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker { + worker.start(py, &name, resource_kind)?; + } if let Some(projector) = &callbacks.projector { let options: Bound<'_, PyDict> = PyDict::new(py); options.set_item("resource_name", name)?; @@ -579,10 +582,15 @@ impl BuildProgressCallbacks { fn on_node_complete(slf: &Bound<'_, Self>, node_result: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = slf.py(); let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker + && let Some((_, key)) = resource_identity(node_result)? + { + worker.completed(node_result, &key)?; + } let kind: ResultKind = result_kind(node_result)?; let mut result: Bound<'_, PyAny> = node_result.clone(); if let Some(projector) = &callbacks.projector - && let Some((name, identity)) = resource_identity(node_result, kind)? + && let Some((name, identity)) = resource_identity(node_result)? { let Some(duration_ms) = consume_terminal(projector.bind(py), &name, Some(&identity))? else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index f92b9941c0..7c0ebc870f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -23,11 +23,13 @@ use crate::bindings::_helpers::init_command::command::run_init_command; use crate::bindings::_helpers::janitor::janitor_command::run_janitor_command; use crate::bindings::_helpers::lineage::lineage_command::run_lineage_command; use crate::bindings::_helpers::load::load_command::run_load_command; +use crate::bindings::_helpers::location_command::command::run_location; use crate::bindings::_helpers::plan_command::plan::run_plan_command; use crate::bindings::_helpers::plan_command::rules::run_rules_command; use crate::bindings::_helpers::query_command::dag::run_dag_command; use crate::bindings::_helpers::query_command::query::run_query_command; use crate::bindings::_helpers::recall::command::run_recall_command; +use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::refactor::refactor_command::run_refactor_command; use crate::bindings::_helpers::review_command::command::run_review_command; @@ -74,7 +76,7 @@ const RAW_QUERY_ATTRIBUTES: [&str; 4] = [ ]; /// One invocation being dispatched: its namespace and the values derived from it once. -struct Invocation<'py> { +pub(crate) struct Invocation<'py> { args: Bound<'py, PyAny>, project_dir: Bound<'py, PyAny>, select: Bound<'py, PyTuple>, @@ -82,6 +84,27 @@ struct Invocation<'py> { } impl<'py> Invocation<'py> { + pub(crate) fn new(args: &Bound<'py, PyAny>) -> PyResult { + let raw_project_dir = native_field(args, "project_dir")?; + let project_dir = if raw_project_dir.is_none() { + raw_project_dir + } else { + path_class(args.py())? + .call1((raw_project_dir,))? + .call_method0("resolve")? + }; + let (select, selector_files) = selector_inputs(args)?; + if native_field(args, "select_file")?.is_truthy()? { + withhold_launch_record(args.py())?; + } + Ok(Self { + args: args.clone(), + project_dir, + select, + selector_files, + }) + } + fn py(&self) -> Python<'py> { self.args.py() } @@ -95,7 +118,7 @@ impl<'py> Invocation<'py> { } /// The value of one handler argument or request field. - fn value(&self, source: Source) -> PyResult> { + pub(crate) fn value(&self, source: Source) -> PyResult> { let py: Python<'py> = self.py(); match source { Source::Attr(name) => self.attr(name), @@ -312,6 +335,7 @@ fn call_route<'py>( invocation: &Invocation<'py>, route: &Route, handler: Option>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = invocation.py(); let mut positional: Vec> = Vec::new(); @@ -339,7 +363,7 @@ fn call_route<'py>( } match handler { Some(handler) => handler.call(PyTuple::new(py, positional)?, Some(&keywords)), - None => run_implementation(route.implementation, (positional, keywords)), + None => run_implementation(route.implementation, (positional, keywords), recall), } } @@ -358,6 +382,7 @@ fn argument<'py>( fn run_implementation<'py>( name: &str, arguments: (Vec>, Bound<'py, PyDict>), + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = arguments.1.py(); let request = || -> PyResult> { @@ -387,6 +412,7 @@ fn run_implementation<'py>( "run_janitor_command" => run_janitor_command(py, &request()?)?, "run_lineage_command" => run_lineage_command(py, &request()?)?, "run_load_command" => run_load_command(py, &request()?)?, + "run_location_command" => run_location(py, options(), Some(recall))?, "run_plan_command" => run_plan_command(&request()?)?, "run_playground_command" => run_playground_command(py, &request()?)?, "run_query_command" => run_query_command(py, options())?, @@ -417,26 +443,10 @@ fn run_implementation<'py>( fn routed_command<'py>( args: &Bound<'py, PyAny>, handlers: Option<&Bound<'py, PyAny>>, + recall: &InvocationRecall, ) -> PyResult> { let py: Python<'py> = args.py(); - let raw_project_dir: Bound<'py, PyAny> = native_field(args, "project_dir")?; - let project_dir: Bound<'py, PyAny> = if raw_project_dir.is_none() { - raw_project_dir - } else { - path_class(py)? - .call1((raw_project_dir,))? - .call_method0("resolve")? - }; - let (select, selector_files) = selector_inputs(args)?; - if native_field(args, "select_file")?.is_truthy()? { - withhold_launch_record(py)?; - } - let invocation = Invocation { - args: args.clone(), - project_dir, - select, - selector_files, - }; + let invocation = Invocation::new(args)?; match command_dispatch(&command_facts(args)?) { Dispatch::Nothing => 0.into_bound_py_any(py), Dispatch::Fail(failure) => Err(user_error(py, failure)), @@ -456,7 +466,7 @@ fn routed_command<'py>( if let Some(failure) = failure { return Err(user_error(py, failure)); } - call_route(&invocation, route, handler) + call_route(&invocation, route, handler, recall) } } } @@ -465,7 +475,14 @@ fn routed_command<'py>( pub(crate) fn dispatch_command<'py>( args: &Bound<'py, PyAny>, handlers: Option>, + recall: &InvocationRecall, ) -> PyResult> { + within(&warehouse_scope(args)?, |_| { + routed_command(args, handlers.as_ref(), recall) + }) +} + +pub(crate) fn warehouse_scope<'py>(args: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = args.py(); let warehouse: Bound<'py, PyAny> = native_field(args, "warehouse")?; if !warehouse.is_none() { @@ -484,7 +501,7 @@ pub(crate) fn dispatch_command<'py>( ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; } - let scope: Bound<'py, PyAny> = Bound::new( + Ok(Bound::new( py, command_warehouse_scope( &native_field(args, "command")?, @@ -492,8 +509,7 @@ pub(crate) fn dispatch_command<'py>( Some(&native_field(args, "vars")?), )?, )? - .into_any(); - within(&scope, |_| routed_command(args, handlers.as_ref())) + .into_any()) } /// The parsed values that select a route, read from the namespace. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 3962b79206..fd3e005f42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -17,6 +17,7 @@ use sqlbuild_cli::dispatch::main::invocation_policy::invocation_policy; use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPolicy}; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost}; +use sqlbuild_orchestration::runs::models::RunOutcome; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords, within}; use crate::bindings::_helpers::build_progress::transient_lines::shared_transient_line_coordinator; @@ -122,9 +123,7 @@ fn print_object(value: &Bound<'_, PyAny>, stream: Option<&str>, end: Option<&str if let Some(end) = end { keywords.set_item("end", end)?; } - py.import("builtins")? - .getattr("print")? - .call((value,), Some(&keywords))?; + imported(py, "builtins", "print")?.call((value,), Some(&keywords))?; Ok(()) } @@ -547,7 +546,11 @@ fn observed_dispatch<'py>( if let Some(session) = &invocation.recall.session { session.publish_context(false); } - return dispatch_command(&invocation.args, invocation.handlers.clone()); + return dispatch_command( + &invocation.args, + invocation.handlers.clone(), + &invocation.recall, + ); } let command: String = invocation.command_name(); drop(take_failure_cause()); @@ -603,11 +606,14 @@ fn observed_dispatch<'py>( "invocation_started", started_payload()?, )?)?; - let dispatched: PyResult> = - match dispatch_command(&invocation.args, invocation.handlers.clone()) { - Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), - dispatched => dispatched, - }; + let dispatched: PyResult> = match dispatch_command( + &invocation.args, + invocation.handlers.clone(), + &invocation.recall, + ) { + Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), + dispatched => dispatched, + }; match dispatched { Ok(exit_code) => { let payload: Bound<'py, PyDict> = PyDict::new(py); @@ -766,6 +772,33 @@ fn parsed_invocation<'py>( Ok(exit_code) }; } + let namespace = args.getattr("__dict__")?.cast_into::()?; + let command: Option = namespace + .get_item("command")? + .map(|value| value.extract::>()) + .transpose()? + .flatten(); + let location_command: Option = namespace + .get_item("location_command")? + .map(|value| value.extract::>()) + .transpose()? + .flatten(); + if command.as_deref() == Some("location") && location_command.as_deref() == Some("runtime") { + return within(&host::redirect_stdout_to_stderr(py)?, |_| { + execute_parsed_invocation(args, handlers, identity, use_color) + }); + } + execute_parsed_invocation(args, handlers, identity, use_color) +} + +/// Runtime stream routing must cover extension discovery, invocation scopes and their teardown. +fn execute_parsed_invocation<'py>( + args: Bound<'py, PyAny>, + handlers: Option>, + identity: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let py = args.py(); let facts: CommandFacts = command_facts(&args)?; let policy: InvocationPolicy = invocation_policy(&facts); let recall: InvocationRecall = if policy.recalled { @@ -841,6 +874,30 @@ fn parsed_invocation<'py>( Ok(_) | Err(_) => None, }; let finished: PyResult<()> = finish_invocation_recall(py, &recall, exit_status); + let worker = recall + .worker + .lock() + .map_err(|_| { + pyo3::exceptions::PyRuntimeError::new_err("worker completion lock is unavailable") + })? + .take(); + if let Some(worker) = worker { + let code = match &finished { + Ok(()) => exit_status.unwrap_or(1), + Err(_) => 1, + }; + let outcome = worker.finish(py, i32::try_from(code).unwrap_or(1))?; + host::write_console( + py, + "stderr", + &format!("Worker execution completed: {}\n", outcome.as_str()), + )?; + if !matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { + dispatched?; + finished?; + return exit(py, code.max(1)); + } + } let exit_code: Bound<'py, PyAny> = dispatched?; finished?; Ok(exit_code) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 6a26b63e16..0f48c13014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -15,7 +15,7 @@ use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords, within}; use crate::bindings::_helpers::build_progress::progress::NativeProgressProjector; use crate::bindings::_helpers::build_progress::progress::machine_output_mode; use crate::bindings::_helpers::cli::launch_state::withhold_launch_record; @@ -337,10 +337,17 @@ impl Observed { ) -> PyResult<()> { let mut command_output_sinks: bool = false; if native_field(args, "command")?.ne(LINEAGE_COMMAND)? { - let extensions: Bound<'_, PyAny> = call_with( - &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, - &[("project_dir", project_dir.clone())], - )?; + let discover = || { + call_with( + &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, + &[("project_dir", project_dir.clone())], + ) + }; + let extensions: Bound<'_, PyAny> = if machine_output_mode(args)? { + within(&host::redirect_stdout_to_stderr(py)?, |_| discover())? + } else { + discover()? + }; let lifecycle_replacement: Bound<'_, PyAny> = native_field(&extensions, "recall_lifecycle_sink")?; let output_replacement: Bound<'_, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 8a824ae6e6..0f6a0d3127 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -189,18 +189,8 @@ fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny anchored.push(entry); continue; }; - let local_file: bool = match mapping.get_item(ATTACH_TYPE_KEY)? { - None => true, - Some(kind) => kind - .extract::() - .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), - }; let copy: Bound<'_, PyDict> = mapping.copy()?; - if let Some(path_value) = mapping.get_item(ATTACH_PATH_KEY)? - && local_file - && !path_value.extract::().is_ok_and(|text| { - text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) - }) + if let Some(path_value) = local_attach_file(mapping)? && let Some(file) = anchored_file_path(&path_value, project_dir)? { copy.set_item(ATTACH_PATH_KEY, file)?; @@ -210,6 +200,59 @@ fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny config.set_item(ATTACH_KEY, pyo3::types::PyList::new(py, anchored)?) } +fn local_attach_file<'py>(mapping: &Bound<'py, PyDict>) -> PyResult>> { + let local = match mapping.get_item(ATTACH_TYPE_KEY)? { + None => true, + Some(kind) => kind + .extract::() + .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), + }; + Ok(mapping.get_item(ATTACH_PATH_KEY)?.filter(|value| { + local + && !value.extract::().is_ok_and(|text| { + text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) + }) + })) +} + +/// Use the resolved adapter connection, including local attachments, not authored TOML guesses. +pub(crate) fn local_database_files( + adapter: &str, + config: &Bound<'_, PyDict>, +) -> PyResult> { + let Some((_, key)) = LOCAL_DATABASE_PATH_KEYS + .iter() + .find(|(name, _)| *name == adapter) + else { + return Ok(Vec::new()); + }; + let mut files: Vec = Vec::new(); + if let Some(value) = config.get_item(*key)? + && let Ok(file) = value.extract::() + && !file.is_empty() + && file != DUCKDB_MEMORY_DATABASE + && !file.contains(URL_SCHEME_SEPARATOR) + && !file.starts_with(MOTHERDUCK_PATH_PREFIX) + { + files.push(file); + } + if adapter == DUCKDB_ADAPTER + && let Some(entries) = config.get_item(ATTACH_KEY)? + && let Ok(entries) = entries.try_iter() + { + for entry in entries { + if let Ok(mapping) = entry?.cast_into::() + && let Some(value) = local_attach_file(&mapping)? + && let Ok(file) = value.extract::() + && file != DUCKDB_MEMORY_DATABASE + { + files.push(file); + } + } + } + Ok(files) +} + /// Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse. #[pyfunction] #[pyo3(signature = (**options))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs new file mode 100644 index 0000000000..a472d4d696 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -0,0 +1,5 @@ +//! Frozen execution protocol values and bounded acknowledgement retries. + +pub(crate) const SKIP_ACTION: &str = "skip"; +pub(crate) const NO_CHANGE_REASON: &str = "no_change"; +pub(crate) const REPORT_ATTEMPTS: usize = 3; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index bbd4e1a61a..2101d9ab7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,6 +1,14 @@ pub(crate) mod attachment_inputs; +pub(crate) mod constants; pub(crate) mod project; pub(crate) mod registration; pub(crate) mod rows; pub(crate) mod schema_facts; +pub(crate) mod snapshot; +pub(crate) mod snapshot_boundary; +pub(crate) mod snapshot_declarations; +pub(crate) mod snapshot_execution; +pub(crate) mod snapshot_models; +pub(crate) mod snapshot_resources; +pub(crate) mod snapshot_worker; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index fd619ffe5f..2c8daa3464 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -6,5 +6,7 @@ use crate::bindings::_helpers::{compiled_views, input_loops}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; input_loops::registration::register(module)?; - compiled_views::registration::register(module) + compiled_views::registration::register(module)?; + crate::bindings::_helpers::location_command::command::register(module)?; + crate::bindings::_helpers::compiled_project::snapshot::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs new file mode 100644 index 0000000000..8edfb3a580 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs @@ -0,0 +1,174 @@ +//! Project metadata stays native until its immutable snapshot is sent over HTTP. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use serde_json::Value; +use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; + +use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ + compile_declarations, discover_declarations, +}; +use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; +use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; +use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; +use crate::bindings::_helpers::records::fields::field; + +/// A data-only publication; this object never contains adapters, connections or user callables. +#[pyclass(frozen, module = "sqlbuild._native")] +pub(crate) struct NativeOrchestrationSnapshot { + pub(crate) input: SnapshotInput, +} + +impl NativeOrchestrationSnapshot { + /// Native callers retain transport errors so they can report an uncertain acknowledgement. + pub(crate) fn publication( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult> { + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Ok(result) + }) + } +} + +#[pymethods] +impl NativeOrchestrationSnapshot { + /// Serialize only at the public boundary; native transports can use the retained input. + fn to_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| serde_json::to_string(&self.input)) + .map_err(|error| PyValueError::new_err(error.to_string())) + } + + /// Publish retained data without projecting it through Python, releasing the GIL during IO. + fn publish(&self, py: Python<'_>, server_url: &str, bearer: &str) -> PyResult { + let record = self + .publication(py, server_url, bearer)? + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + py.detach(|| { + serde_json::to_string(&record) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Retain the verified claim and its independent native heartbeat in the worker interpreter. + fn start_worker( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + identity: (String, String), + ) -> PyResult { + NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) + } +} + +/// Compile catalog nodes from their owners, never from Python model or graph projections. +#[pyfunction] +#[pyo3(signature = (project, metadata_json, project_dir, *, declarations=None))] +pub(crate) fn build_orchestration_snapshot( + project: PyRef<'_, NativeProjectView>, + metadata_json: &str, + project_dir: PathBuf, + declarations: Option>>, +) -> PyResult { + let py = project.py(); + let mut input: SnapshotInput = serde_json::from_str(metadata_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !input.nodes.is_empty() { + return Err(PyValueError::new_err( + "snapshot nodes must come from the compiled project", + )); + } + if project + .effective_target_name() + .is_some_and(|target| target != input.target) + { + return Err(PyValueError::new_err( + "snapshot target differs from the compiled project", + )); + } + for diagnostic in project.diagnostics(py).bind(py).try_iter()? { + let severity: String = field(&field(&diagnostic?, "severity")?, "value")?.extract()?; + if severity == DiagnosticSeverity::Error.as_str() { + return Err(PyValueError::new_err( + "cannot publish a project with compiler errors", + )); + } + } + input.sqb_version = env!("CARGO_PKG_VERSION").into(); + input.environment_hash = environment_hash(py, &project_dir)?; + let owner = project.owner().bind(py).borrow(); + input.nodes = model_nodes(py, &project, &owner, input.nodes)?; + input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; + drop(owner); + let declarations = match declarations { + Some(declarations) => declarations, + None => discover_declarations(py, &project_dir)?, + }; + if !declarations.is_empty() { + let authored = collect_declarations(py, declarations)?; + let graph = project.lineage_graph(py)?; + input = compile_declarations( + py, + input, + (&project_dir, graph.bind(py).get().graph()), + authored, + )?; + } + Ok(NativeOrchestrationSnapshot { input }) +} + +/// Duplicate resource identities must fail, rather than silently replacing an earlier node. +pub(crate) fn insert_node( + mut nodes: BTreeMap, + key: String, + node: SnapshotNode, +) -> PyResult> { + if nodes.insert(key, node).is_some() { + return Err(PyValueError::new_err("duplicate snapshot node identity")); + } + Ok(nodes) +} + +pub(crate) fn dependency_keys(deps: &[(String, String)]) -> Vec { + deps.iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect() +} + +pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> Value { + serde_json::json!({ + "database": value.database, + "schema": value.schema, + "name": value.name, + "qualified_name": value.qualified_name, + "logical_database": value.logical_database, + "logical_schema": value.logical_schema, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_orchestration_snapshot, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs new file mode 100644 index 0000000000..8eaa09494a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs @@ -0,0 +1,183 @@ +//! Discovery files and authored test payloads remain the Python host's boundary. + +use std::collections::BTreeMap; +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{destination, insert_node}; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::compiled_views::seed_views::{NativeCompiledSeed, SeedViews}; +use crate::bindings::_helpers::records::fields::{field, path_posix}; + +/// Read only declared metadata fields, never callable attributes or connection settings. +pub(crate) fn metadata(value: &Bound<'_, PyAny>, names: &[&str]) -> PyResult { + let mut result = Map::new(); + for name in names { + result.insert((*name).into(), plain_value(&field(value, name)?)?); + } + Ok(Value::Object(result)) +} + +pub(crate) fn relative_path(value: &Bound<'_, PyAny>) -> PyResult { + path_posix(&field(value, "relative_path")?) +} + +/// Source locations are local compiler diagnostics, not relocatable column metadata. +pub(crate) fn columns(value: &Bound<'_, PyAny>) -> PyResult> { + value + .try_iter()? + .map(|column| { + metadata( + &column?, + &["name", "type", "nullable", "description", "meta"], + ) + }) + .collect() +} + +fn file_digest(value: &Bound<'_, PyAny>) -> PyResult { + let path: PathBuf = field(value, "file_path")?.extract()?; + value.py().detach(|| { + let mut file = File::open(path)?; + let mut hash = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let length = file.read(&mut buffer)?; + if length == 0 { + return Ok(format!("{:x}", hash.finalize())); + } + hash.update(&buffer[..length]); + } + }) +} + +pub(crate) fn boundary_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + let seed_views = project.seeds(py); + for (seed, view) in owner.seeds().iter().zip(seed_views.bind(py).try_iter()?) { + let effective = view? + .cast::()? + .get() + .current_destination(py)?; + let parts = seed.parts(); + let seed_file = parts.seed_file.bind(py); + nodes = insert_node( + nodes, + format!("seed:{}", seed.name), + SnapshotNode { + kind: "seed".into(), + definition: json!({ + "name": seed.name, + "path": relative_path(seed_file)?, + "content_hash": file_digest(seed_file)?, + "destination": destination(&effective), + "schema": metadata(parts.schema_entry.bind(py), &[ + "tags", "description", "meta", "csv_settings", + ])?, + "columns": columns(&field(parts.schema_entry.bind(py), "columns")?)?, + }), + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + )?; + } + for (kind, objects, names) in [ + ( + "sql_test", + project.sql_tests(py), + ["mode", "payload", "parameter_schema", "parameter_values"].as_slice(), + ), + ( + "sql_scenario", + project.sql_scenarios(py), + ["authored_ctes", "expected_ctes", "assertion_ctes"].as_slice(), + ), + ] { + for item in objects.bind(py).try_iter()? { + let item = item?; + let key = field(&item, "key")?; + let name: String = field(&key, "name")?.extract()?; + let resource_type: String = field(&key, "resource_type")?.str()?.extract()?; + let definition = metadata(&item, names)?; + nodes = insert_node( + nodes, + format!("{resource_type}:{name}"), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": name, "test": definition}), + raw_sql: None, + compiled_sql: Some(field(&item, "sql_body")?.extract()?), + python_source: None, + }, + )?; + } + } + for (kind, objects) in [ + ("loader", project.loader_functions(py)), + ("hook", project.hook_functions(py)), + ("materialization", project.materialization_files(py)), + ] { + for item in objects.bind(py).try_iter()? { + let item = item?; + let name: String = field(&item, "name")?.extract()?; + let path: PathBuf = field(&item, "file_path")?.extract()?; + let source = py.detach(|| std::fs::read_to_string(path))?; + nodes = insert_node( + nodes, + format!("{kind}:{name}"), + SnapshotNode { + kind: kind.into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: None, + compiled_sql: None, + python_source: Some(source), + }, + )?; + } + } + for item in project.sql_hook_files(py).bind(py).try_iter()? { + let item = item?; + let name: String = field(&item, "name")?.extract()?; + nodes = insert_node( + nodes, + format!("sql_hook:{name}"), + SnapshotNode { + kind: "sql_hook".into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: Some(field(&item, "contents")?.extract()?), + compiled_sql: Some(field(&item, "sql_body")?.extract()?), + python_source: None, + }, + )?; + } + let macros = project.loaded_macros(py); + for (key, item) in macros.bind(py).cast::()?.iter() { + let name: String = key.extract()?; + nodes = insert_node( + nodes, + format!("macro:{name}"), + SnapshotNode { + kind: "macro".into(), + definition: json!({"name": name, "path": relative_path(&item)?}), + raw_sql: None, + compiled_sql: None, + python_source: Some(field(&item, "raw_source")?.extract()?), + }, + )?; + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs new file mode 100644 index 0000000000..c04d8a5435 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -0,0 +1,241 @@ +//! Compile authoring records against retained graph data before publication. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use serde_json::Value; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; +use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; +use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; +use sqlbuild_orchestration::declarations::models::{ + Definitions, GroupDefinition, JobDefinition, OnFailure, TriggerDeclaration, TriggerDefinition, +}; +use sqlbuild_orchestration::scheduler::models::Overlap; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; + +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, module_jobs, +}; +use crate::bindings::_helpers::native_host::orchestration_triggers::{ + compile_calendars, compile_cron, compile_sensors, +}; + +/// Dedicated declaration modules are imported only in the code-location interpreter. +pub(crate) fn discover_declarations<'py>( + py: Python<'py>, + project_dir: &Path, +) -> PyResult>> { + let tree = ProjectTree::new(project_dir); + let files = py + .detach(|| { + rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { + !entry.is_dir + && entry.name.ends_with(PYTHON_FILE_SUFFIX) + && entry.name.strip_suffix(PYTHON_FILE_SUFFIX) != Some(PYTHON_INIT_MODULE_STEM) + }) + }) + .map_err(|error| { + PyValueError::new_err(format!("orchestration discovery failed: {error:?}")) + })?; + let mut declarations: Vec> = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + for file in files { + for job in module_jobs(py, &tree.absolute(&file), project_dir)? { + if seen.insert(job.as_ptr() as usize) { + declarations.push(job); + } + } + } + Ok(declarations) +} + +pub(crate) fn compile_declarations( + py: Python<'_>, + mut input: SnapshotInput, + (project_dir, graph): (&Path, &ProjectGraph), + declarations: Vec, +) -> PyResult { + if declarations.is_empty() { + return Ok(input); + } + if ["jobs", "groups", "triggers"] + .iter() + .any(|key| input.documents.contains_key(*key)) + { + return Err(PyValueError::new_err( + "authored declarations cannot overwrite snapshot job or trigger documents", + )); + } + let mut jobs: BTreeMap = BTreeMap::new(); + let mut authored_groups: Vec = Vec::new(); + let mut authored_jobs: Vec = Vec::new(); + let mut authored_calendars: Vec> = Vec::new(); + let mut authored_sensors: Vec = Vec::new(); + for declaration in declarations { + match declaration { + AuthoredDeclaration::Job(job) => authored_jobs.push(job), + AuthoredDeclaration::Group(group) => authored_groups.push(group), + AuthoredDeclaration::Calendar(calendar) => authored_calendars.push(calendar), + AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor), + } + } + let mut triggers: BTreeMap = BTreeMap::new(); + let mut names: BTreeSet = BTreeSet::new(); + for job in authored_jobs { + if !names.insert(job.name.clone()) { + return Err(PyValueError::new_err(format!("duplicate job {}", job.name))); + } + if !job.targets.is_empty() && !job.targets.contains(&input.target) { + continue; + } + let mut settings: BTreeMap = serde_json::from_value(job.settings) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + if !job.concurrency.is_null() { + if settings.contains_key("concurrency") { + return Err(PyValueError::new_err("job concurrency is declared twice")); + } + settings.insert("concurrency".into(), job.concurrency); + } + job_settings_arguments(&settings).map_err(PyValueError::new_err)?; + let tokens = |value: &str| { + value + .split_whitespace() + .map(str::to_owned) + .collect::>() + }; + if job.select.trim().is_empty() { + return Err(PyValueError::new_err(format!( + "job {} needs a selection", + job.name + ))); + } + let selection: BTreeSet = + resolve_selectors(graph, &tokens(&job.select), &tokens(&job.exclude)) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if let Some(schedule) = job.schedule { + triggers.insert( + job.name.clone(), + compile_cron(schedule.bind(py), &job.name, &job.description)?, + ); + } + jobs.insert( + job.name, + JobDefinition { + description: job.description, + selection, + settings, + overlap: serde_json::from_value::(Value::String(job.overlap)) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + on_failure: serde_json::from_value::(Value::String(job.on_failure)) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + concurrency_groups: job.concurrency_groups.into_iter().collect(), + }, + ); + } + let mut groups: BTreeMap = BTreeMap::new(); + for group in authored_groups { + if !names.insert(group.name.clone()) { + return Err(PyValueError::new_err(format!( + "duplicate job or group {}", + group.name + ))); + } + if !group.targets.is_empty() && !group.targets.contains(&input.target) { + continue; + } + let members: BTreeSet = group.jobs.iter().map(|job| job.name.clone()).collect(); + if members.len() != group.jobs.len() { + return Err(PyValueError::new_err("group members must be distinct")); + } + let mut after: BTreeMap> = BTreeMap::new(); + for (job, dependencies) in group.after { + after.insert( + job.name, + dependencies + .into_iter() + .map(|dependency| dependency.name) + .collect(), + ); + } + let dependencies = group_dependencies(&input, &jobs, (&members, &after)) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let selection: BTreeSet = members + .iter() + .flat_map(|name| jobs[name].selection.iter().cloned()) + .collect(); + if let Some(schedule) = group.schedule { + triggers.insert( + group.name.clone(), + compile_cron(schedule.bind(py), &group.name, &group.description)?, + ); + } + groups.insert( + group.name, + GroupDefinition { + job: JobDefinition { + description: group.description, + selection, + settings: BTreeMap::new(), + overlap: serde_json::from_value(Value::String(group.overlap)) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + on_failure: OnFailure::SkipDependents, + concurrency_groups: group.concurrency_groups.into_iter().collect(), + }, + jobs: dependencies, + }, + ); + } + let Definitions { + jobs, + groups, + triggers, + sensors, + } = compile_sensors( + py, + (project_dir, input.target.as_str(), graph), + authored_sensors, + compile_calendars( + py, + (project_dir, &input.target), + authored_calendars, + Definitions { + jobs, + groups, + triggers, + ..Definitions::default() + }, + )?, + )?; + let triggers: BTreeMap<_, _> = triggers + .into_iter() + .map(|(name, trigger)| (name, TriggerDeclaration::Schedule(trigger))) + .chain( + sensors + .into_iter() + .map(|(name, sensor)| (name, TriggerDeclaration::Sensor(sensor))), + ) + .collect(); + for (key, document) in [ + ("jobs", serde_json::to_value(jobs)), + ("groups", serde_json::to_value(groups)), + ("triggers", serde_json::to_value(triggers)), + ] { + input.documents.insert( + key.into(), + document.map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + } + snapshot_definitions(&input).map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok(input) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs new file mode 100644 index 0000000000..a7aa2f894c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -0,0 +1,260 @@ +//! Frozen selection, executor callbacks and completion belong to one verified worker build. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyStringMethods; +use sqlbuild_core::models::ResourceKind; +use sqlbuild_orchestration::runs::models::{NodeOutcome, RunChange, RunExecution, RunOutcome}; + +use crate::bindings::_helpers::compiled_project::constants::{NO_CHANGE_REASON, SKIP_ACTION}; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::records::fields::field; + +/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. +#[derive(Debug)] +pub(crate) struct WorkerBuild { + pub(crate) worker: Py, + execution: RunExecution, + completed: Mutex>, +} + +impl WorkerBuild { + pub(crate) fn new(worker: &Bound<'_, NativeOrchestrationWorker>) -> PyResult> { + let state = Arc::new(Self { + execution: worker.get().execution(worker.py())?, + worker: worker.clone().unbind(), + completed: Mutex::default(), + }); + for key in state.execution.skipped_nodes.keys() { + state.complete(worker.py(), key, (NodeOutcome::Skipped, None, None))?; + } + Ok(state) + } + + pub(crate) fn has_work(&self) -> bool { + self.execution + .claim + .request + .selection + .iter() + .any(|key| !self.execution.skipped_nodes.contains_key(key)) + } + + pub(crate) fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { + if self.worker.bind(py).get().cancel_requested(py) { + return Err(PyKeyboardInterrupt::new_err("run cancellation requested")); + } + Ok(()) + } + + /// Reject an implicitly expanded write set before any executor connection or schema change. + pub(crate) fn enforce_plan_writers( + &self, + pipeline: &Bound<'_, PyAny>, + ) -> Result<(), pyo3::PyErr> { + self.check_cancelled(pipeline.py())?; + let plan = field(pipeline, "plan_output")?; + for collection in [ + "model_entries", + "seed_entries", + "function_entries", + "source_load_entries", + ] { + for entry in field(&plan, collection)?.try_iter()? { + let entry = entry?; + let identity = field(&entry, "key")?; + let kind = field(&identity, "resource_type")? + .str()? + .to_str()? + .to_owned(); + let key = format!("{kind}:{}", field(&identity, "name")?.str()?); + if !self.execution.claim.request.write_set.contains(&key) { + return Err(PyValueError::new_err(format!( + "planned writer {key} is outside the frozen run write set" + ))); + } + if kind == ResourceKind::Model.as_str() + && field(&entry, "action")?.str()?.to_str()? == SKIP_ACTION + { + let outcome = if field(&entry, "reason")?.str()?.to_str()? == NO_CHANGE_REASON { + NodeOutcome::Success + } else { + NodeOutcome::Skipped + }; + self.complete(pipeline.py(), &key, (outcome, None, None))?; + } + } + } + Ok(()) + } + + fn complete( + &self, + py: Python<'_>, + key: &str, + (outcome, rows_affected, error_code): (NodeOutcome, Option, Option), + ) -> PyResult<()> { + if !self.execution.claim.request.selection.contains(key) { + return Ok(()); + } + if self + .completed + .lock() + .map_err(|_| poisoned())? + .contains_key(key) + { + return Ok(()); + } + self.worker.bind(py).get().change( + py, + &format!("node-finished/{key}"), + RunChange::NodeFinished { + node_key: key.into(), + outcome, + rows_affected, + error_code, + skipped_because_of: self.execution.skipped_nodes.get(key).cloned(), + }, + )?; + self.completed + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), outcome); + Ok(()) + } + + pub(crate) fn start( + &self, + py: Python<'_>, + name: &str, + resource_kind: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.check_cancelled(py)?; + let kind = field(resource_kind, "value")?.extract::()?; + let kind = match kind.as_str() { + "table" | "view" | "snapshot" | "custom" => "model", + kind => kind, + }; + let key = format!("{kind}:{name}"); + if self.execution.claim.request.selection.contains(&key) { + self.worker.bind(py).get().change( + py, + &format!("node-started/{key}"), + RunChange::NodeStarted { node_key: key }, + )?; + } + Ok(()) + } + + /// Existing native progress callbacks publish selected-node evidence without Python relays. + pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + if self.execution.claim.request.selection.contains(key) { + let status: String = field(result, "status")?.str()?.extract()?; + let outcome = match status.as_str() { + "success" => { + if result.hasattr("warning_messages")? + && field(result, "warning_messages")?.is_truthy()? + { + NodeOutcome::Warn + } else { + NodeOutcome::Success + } + } + "failed" => { + if self + .worker + .bind(result.py()) + .get() + .refresh_cancellation(result.py())? + { + NodeOutcome::Cancelled + } else { + NodeOutcome::Failed + } + } + "skipped" => NodeOutcome::Skipped, + _ => { + return Err(PyValueError::new_err(format!( + "unsupported worker node outcome {status}" + ))); + } + }; + let rows = match result.getattr_opt("rows_affected")? { + Some(value) => value.extract()?, + None => None, + }; + let error = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + self.complete(result.py(), key, (outcome, rows, error))?; + } + Ok(()) + } + + /// Never call missing selected nodes successful merely because the executor returned zero. + pub(crate) fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; + for key in &self.execution.claim.request.selection { + self.complete( + py, + key, + ( + if cancelled { + NodeOutcome::Cancelled + } else { + NodeOutcome::Skipped + }, + None, + None, + ), + )?; + } + let outcome = { + let completed = self.completed.lock().map_err(|_| poisoned())?; + let good = completed + .values() + .any(|value| matches!(value, NodeOutcome::Success | NodeOutcome::Warn)); + let bad = completed.values().any(|value| { + matches!( + value, + NodeOutcome::Failed | NodeOutcome::Skipped | NodeOutcome::Cancelled + ) + }); + if cancelled { + RunOutcome::Cancelled + } else if bad && good { + RunOutcome::Partial + } else if bad || exit_code != 0 { + RunOutcome::Failed + } else if completed.values().any(|value| *value == NodeOutcome::Warn) { + RunOutcome::Warn + } else { + RunOutcome::Success + } + }; + Ok(outcome) + } + + /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. + pub(crate) fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { + let outcome = self.outcome(py, exit_code)?; + self.worker.bind(py).get().change( + py, + "worker-finished", + RunChange::Finished { + outcome, + reason: "worker executor completed and closed its connections".into(), + exit_code: Some(i64::from(exit_code)), + }, + )?; + Ok(outcome) + } +} + +fn poisoned() -> pyo3::PyErr { + PyRuntimeError::new_err("worker build outcome lock is unavailable") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs new file mode 100644 index 0000000000..13fa7aca8c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs @@ -0,0 +1,73 @@ +//! Model catalog data comes directly from the compiler's retained facts and records. + +use std::collections::BTreeMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyResult, Python}; +use pyo3::types::PyTupleMethods; +use serde_json::{Value, json}; +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{ + dependency_keys, destination, insert_node, +}; +use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; + +pub(crate) fn model_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + for model in project.models(py).bind(py).iter() { + let model = model.cast::()?.get(); + let (name, _, _) = model.file_texts(py)?; + let facts = model_facts(owner.facts(), &name) + .ok_or_else(|| PyValueError::new_err("snapshot model has no retained facts"))?; + if facts.config.encoding_error.is_some() { + return Err(PyValueError::new_err( + "model config cannot be represented in snapshot metadata", + )); + } + let mut definition = model.read(py, |input, _| { + let record = input + .compiled + .as_ref() + .ok_or_else(|| PyValueError::new_err("snapshot model has not been assembled"))?; + let columns: Vec = record + .analysis + .inferred + .as_deref() + .unwrap_or_default() + .iter() + .map(|(name, data_type, nullability)| { + json!({"name": name, "type": data_type, "nullability": nullability}) + }) + .collect(); + Ok(json!({ + "name": facts.name, + "path": facts.relative_path, + "config": facts.config.values, + "materialization": facts.config.materialized, + "deps": dependency_keys(&record.deps), + "columns": columns, + })) + })?; + definition["destination"] = model.read_destination(py, |value| Ok(destination(value)))?; + nodes = insert_node( + nodes, + format!("model:{}", facts.name), + SnapshotNode { + kind: "model".into(), + definition, + raw_sql: Some(facts.authored_sql.to_string()), + compiled_sql: Some(facts.query_sql.to_string()), + python_source: None, + }, + )?; + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs new file mode 100644 index 0000000000..213bcc9739 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs @@ -0,0 +1,140 @@ +//! Sources, functions and audits are read from their retained native records. + +use std::collections::BTreeMap; + +use pyo3::prelude::{PyAnyMethods, PyResult, Python}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::snapshot::{ + dependency_keys, destination, insert_node, +}; +use crate::bindings::_helpers::compiled_project::snapshot_boundary::{ + boundary_nodes, columns, metadata, relative_path, +}; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; + +pub(crate) fn resource_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + mut nodes: BTreeMap, +) -> PyResult> { + for source in owner.sources() { + let attached = &source.attached; + nodes = insert_node( + nodes, + format!("source:{}", attached.name), + SnapshotNode { + kind: "source".into(), + definition: json!({ + "name": attached.name, + "path": relative_path(attached.host.source_file.bind(py))?, + "database": attached.database, + "schema": attached.schema, + "table": attached.table, + "expression": attached.expression, + "contract": attached.contract, + "description": attached.description, + "namespace": source.namespace, + "columns": columns(attached.host.columns.bind(py).as_any())?, + "meta": plain_value(attached.host.meta.bind(py))?, + "entry": metadata(attached.host.entry.bind(py), &[ + "loader", "write_strategy", "cursor_column", "unique_key", + "final", "load_batch_size", "freshness", + ])?, + }), + raw_sql: None, + compiled_sql: attached.expression.clone(), + python_source: None, + }, + )?; + } + let function_views = project.functions(py); + for (function, view) in owner + .functions() + .iter() + .zip(function_views.bind(py).try_iter()?) + { + let effective = view? + .cast::()? + .get() + .current_destinations(py)? + .0; + let attached = &function.attached; + let kind = attached.node_type(); + let python = attached.python.as_ref().map(|runtime| { + json!({ + "version": runtime.runtime_version, + "entry_point": runtime.entry_point, + "packages": runtime.packages, + }) + }); + nodes = insert_node( + nodes, + format!("{kind}:{}", attached.name), + SnapshotNode { + kind: kind.into(), + definition: json!({ + "name": attached.name, + "path": attached.relative_path, + "arguments": attached.arguments, + "returns": attached.returns, + "return_columns": attached.return_columns, + "destination": destination(&effective), + "python": python, + "tags": attached.tags, + "description": attached.description, + "deps": dependency_keys(function.deps.as_deref().unwrap_or_default()), + }), + raw_sql: None, + compiled_sql: attached.python.is_none().then(|| attached.body.clone()), + python_source: attached.python.is_some().then(|| attached.body.clone()), + }, + )?; + } + for audit in owner.audits() { + let attached = &audit.attached; + let target: Option = attached.target.as_ref().map( + |target| json!({"kind": target.kind, "name": target.name, "column": target.column}), + ); + nodes = insert_node( + nodes, + format!("audit:{}", attached.compiled_name()), + SnapshotNode { + kind: "audit".into(), + definition: json!({ + "name": attached.compiled_name(), + "definition_name": attached.definition_name, + "measurement": attached.measurement, + "evidence_sql": attached.evidence_sql, + "target": target, + "severity": attached.severity, + "run_scope": attached.run_scope, + "always_run": attached.always_run, + "description": attached.description, + "policy": metadata(attached.host.audit_block.bind(py), &[ + "evaluation_mode", "measurement_contract", + ])?, + "thresholds": plain_value( + attached.host.thresholds.bind(py), + )?, + "minimum_samples": plain_value( + attached.host.minimum_samples.bind(py), + )?, + "evidence_limit": plain_value( + attached.host.evidence_limit.bind(py), + )?, + "deps": dependency_keys(audit.scope_deps.as_deref().unwrap_or_default()), + }), + raw_sql: None, + compiled_sql: Some(attached.sql_body.clone()), + python_source: None, + }, + )?; + } + boundary_nodes(py, project, owner, nodes) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs new file mode 100644 index 0000000000..5c5da15456 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs @@ -0,0 +1,213 @@ +//! A worker interpreter retains the native claim owner; it never opens the control database. + +use std::sync::Mutex; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{PyRef, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_orchestration::runs::models::RunChange; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; +use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; +use uuid::Uuid; + +use crate::bindings::_helpers::compiled_project::constants::REPORT_ATTEMPTS; +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; + +struct WorkerOwner { + runtime: tokio::runtime::Runtime, + client: WorkerClient, + session: WorkerSession, +} + +/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. +#[pyclass(frozen, module = "sqlbuild._native")] +pub(crate) struct NativeOrchestrationWorker { + owner: Mutex>, +} + +impl NativeOrchestrationWorker { + /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. + pub(crate) fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + if owner.session.cancel_requested() { + return Ok(true); + } + owner + .runtime + .block_on(owner.client.heartbeat()) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Native execution reads the retained request, never caller-provided selection or settings. + pub(crate) fn execution( + &self, + py: Python<'_>, + ) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + Ok(owner + .as_ref() + .ok_or_else(unavailable)? + .session + .execution + .clone()) + }) + } + + /// A bounded retry keeps the exact event identity across an uncertain acknowledgement. + pub(crate) fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); + owner.runtime.block_on(async { + let mut attempts = 0; + loop { + attempts += 1; + match owner.client.report(event_id, change.clone()).await { + Ok(_) => return Ok(()), + Err(sqlbuild_orchestration::worker::errors::WorkerError::Transport(_)) + if attempts < REPORT_ATTEMPTS => + { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), + } + } + }) + }) + } + + /// Verify this retained baseline before the caller applies run settings or opens a warehouse. + pub(crate) fn start( + snapshot: &NativeOrchestrationSnapshot, + py: Python<'_>, + (server_url, bearer): (&str, &str), + (run_id, claim_id): (String, String), + ) -> PyResult { + let identity = ( + Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let connected = runtime.block_on(async { + let client = connect_worker(server_url, bearer, identity)?; + let session = start_worker(&client, &snapshot.input).await?; + Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) + }); + match connected { + Ok((client, session)) => Ok(NativeOrchestrationWorker { + owner: Mutex::new(Some(WorkerOwner { + runtime, + client, + session, + })), + }), + Err(error) => { + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Err(PyRuntimeError::new_err(error.to_string())) + } + } + }) + } +} + +#[pymethods] +impl NativeOrchestrationWorker { + /// The frozen request supplies selection and settings only after baseline verification. + fn execution_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + serde_json::to_string(&owner.session.execution) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Cancellation remains sticky; a closed worker cannot authorize further work. + pub(crate) fn cancel_requested(&self, py: Python<'_>) -> bool { + py.detach(|| { + self.owner.lock().map_or(true, |owner| { + owner + .as_ref() + .is_none_or(|owner| owner.session.cancel_requested()) + }) + }) + } + + /// Retry an uncertain report with the identical event ID and change, never a new identity. + fn report(&self, py: Python<'_>, event_id: &str, change_json: &str) -> PyResult { + let event_id = + Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; + let change: RunChange = serde_json::from_str(change_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + let owner = owner.as_ref().ok_or_else(unavailable)?; + owner + .runtime + .block_on(owner.client.report(event_id, change)) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { + if slf.owner.lock().map_err(|_| unavailable())?.is_none() { + return Err(unavailable()); + } + Ok(slf) + } + + fn __exit__( + &self, + py: Python<'_>, + _kind: &pyo3::Bound<'_, pyo3::PyAny>, + _value: &pyo3::Bound<'_, pyo3::PyAny>, + _traceback: &pyo3::Bound<'_, pyo3::PyAny>, + ) -> PyResult<()> { + self.close(py) + } + + /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. + fn close(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| { + let mut owner = self.owner.lock().map_err(|_| unavailable())?; + if let Some(active) = owner.take() { + active.close(); + } + Ok(()) + }) + } +} + +fn unavailable() -> pyo3::PyErr { + PyRuntimeError::new_err("native worker session is closed or unavailable") +} + +impl WorkerOwner { + fn close(self) { + drop(self.session); + drop(self.client); + self.runtime + .shutdown_timeout(std::time::Duration::from_secs(1)); + } +} + +impl Drop for NativeOrchestrationWorker { + /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. + fn drop(&mut self) { + if let Ok(owner) = self.owner.get_mut() + && let Some(owner) = owner.take() + { + owner.close(); + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs index 90a9f140d2..2a551bbd39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs @@ -9,6 +9,7 @@ use serde_json::{Map, Number, Value}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::models::NativeRecord; const DATACLASS_FIELDS: &str = "__dataclass_fields__"; const INSTANCE_DICT: &str = "__dict__"; @@ -49,6 +50,16 @@ pub(crate) fn plain_value(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(mapping) = value.downcast::() { return object_value(mapping); } + if let Ok(record) = value.cast::() { + let record = record.get(); + let mut entries = Vec::with_capacity(record.values.len()); + for (field, item) in record.spec.fields.iter().zip(&record.values) { + if !field.name.starts_with(PRIVATE_PREFIX) { + entries.push((field.name.to_owned(), plain_value(item.bind(value.py()))?)); + } + } + return Ok(sorted_object(entries)); + } object_like_value(value) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs index 441e7d723d..a83881c415 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs @@ -260,6 +260,22 @@ impl NativeCompiledModel { self.read(py, |record, py| read(compiled_of(record)?, py)) } + /// Read the effective destination, including namespace overlays, without a Python projection. + pub(crate) fn read_destination( + &self, + py: Python<'_>, + read: impl FnOnce(&DestinationFact) -> PyResult, + ) -> PyResult { + self.compiled(py, |compiled, _| { + read( + self.overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination), + ) + }) + } + /// This view's typed analysis as the owner's model facts read it, and the model's name. pub(crate) fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { self.read(py, |record, _| { @@ -552,16 +568,11 @@ impl NativeCompiledModel { #[getter] pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, py| { + self.read_destination(py, |destination| { self.projections .destination .get_or_try_init(py, || { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let destination: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); relation_location(&classes.compiled, destination) }) .map(|destination| destination.clone_ref(py)) @@ -612,12 +623,7 @@ impl NativeCompiledModel { /// `(database, schema, name)` of the model's destination. #[getter] pub(crate) fn destination_row(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| { - let destination: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); + self.read_destination(py, |destination| { Ok(( destination.database.clone(), destination.schema.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs index 7be4f6bee8..af49d0e3f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs @@ -436,7 +436,10 @@ impl NativeCompiledFunction { } /// The destination and fingerprint destination, with target defaults once applied. - fn current_destinations(&self, py: Python<'_>) -> PyResult<(DestinationFact, DestinationFact)> { + pub(crate) fn current_destinations( + &self, + py: Python<'_>, + ) -> PyResult<(DestinationFact, DestinationFact)> { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs index b6f857e5f5..2d070f9535 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs @@ -237,7 +237,7 @@ impl NativeCompiledSeed { }) } - fn current_destination(&self, py: Python<'_>) -> PyResult { + pub(crate) fn current_destination(&self, py: Python<'_>) -> PyResult { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index ae20cf9408..c0e0910921 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -28,7 +28,9 @@ use crate::bindings::_helpers::boundary::python_calls::{ }; use crate::bindings::_helpers::cli::failure_cause::{FailureCause, report_failure_cause}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; -use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::command_runtime::connection::{ + local_database_files, project_connection_config, +}; use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, resolution_message, }; @@ -39,6 +41,7 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -357,6 +360,7 @@ fn build_report( ConnectionCheck::Skipped }; Ok(DebugReport { + local_files: local_database_files(&facts.adapter_name, &connection_config)?, runtime: debug_runtime_lines(&runtime(py)?), configuration: debug_configuration_lines(&facts, &warehouses), providers: debug_provider_lines(&providers(&context.discovered_inputs)?), @@ -381,15 +385,23 @@ pub(crate) fn run_debug_command( } else { project_dir.extract()? }; - let report: DebugReport = build_report( - py, - &project_dir, - ( - arguments.value("selected_target")?, - !arguments.flag("no_connection")?, - ), - )?; - let text: String = if arguments.flag("json_output")? { + let json_output = arguments.flag("json_output")?; + let build = || { + build_report( + py, + &project_dir, + ( + arguments.value("selected_target")?, + !arguments.flag("no_connection")?, + ), + ) + }; + let report: DebugReport = if json_output { + within(&redirect_stdout_to_stderr(py)?, |_| build())? + } else { + build()? + }; + let text: String = if json_output { format_debug_json(&report) + "\n" } else { let use_color: bool = !arguments.flag("no_color")? && stdout_supports_color(py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 7e4ed5ab92..6a71912e04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -3,10 +3,10 @@ use fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION; use sqlbuild_analysis::contracts::constants::CONTRACT_ENFORCED; use sqlbuild_analysis::fingerprint_storage::constants::{ - COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, - COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, - NODE_TYPE_MODEL, NODE_TYPE_SEED, + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, + COLUMN_NODE_NAME, COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, + COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, + COLUMN_VERSION_HASH, NODE_TYPE_MODEL, NODE_TYPE_SEED, }; use sqlbuild_analysis::planner_changes::constants::{ AUDIT_GATE_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, @@ -99,6 +99,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("COLUMN_DEFINITION_B64", COLUMN_DEFINITION_B64), ("COLUMN_METADATA_JSON_B64", COLUMN_METADATA_JSON_B64), ("COLUMN_TIMESTAMP", COLUMN_TIMESTAMP), + ("COLUMN_EVENT_ID", COLUMN_EVENT_ID), ("NODE_TYPE_MODEL", NODE_TYPE_MODEL), ("NODE_TYPE_SEED", NODE_TYPE_SEED), ("AUDIT_GATE_METADATA_KEY", AUDIT_GATE_METADATA_KEY), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index e977d6a155..3347752271 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -130,7 +130,7 @@ fn borrowed(identities: &[(String, String)]) -> Vec> { .collect() } -fn row_fingerprint<'py>( +pub(crate) fn row_fingerprint<'py>( fingerprint_class: &Bound<'py, PyAny>, from_iso: &Bound<'py, PyAny>, row: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 3a92672f90..b4612c1c7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::fingerprints::storage_sql::{ use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; @@ -97,8 +98,13 @@ fn create_sql( )) } -/// The table DDL, adapter-rendered when the adapter overrides it, then any index DDL. -fn setup_sqls(location: &Location<'_, '_>, writer: &Writer<'_, '_>) -> PyResult> { +/// Create the table/indexes, then add the nullable identity column to a pre-existing table. +fn setup<'py>( + location: &Location<'_, 'py>, + writer: &Writer<'_, 'py>, + request: &Request<'_, 'py>, + describe: &dyn Fn(&PyErr) -> PyResult>, +) -> PyResult<()> { let py: Python<'_> = writer.execute.py(); let mut sqls: Vec = vec![match writer.render_create_table_sql { Some(render) => render @@ -118,7 +124,31 @@ fn setup_sqls(location: &Location<'_, '_>, writer: &Writer<'_, '_>) -> PyResult< sqls.push(sql?.extract()?); } } - Ok(sqls) + let database = location.database.into_pyobject(py)?.into_any(); + let schema = location.schema.into_pyobject(py)?.into_any(); + retry_write( + py, + || { + for sql in &sqls { + execute_sql(writer, sql)?; + } + ensure_event_id_column( + (writer.connection, writer.execute), + (&database, &schema), + (FINGERPRINT_TABLE_NAME, &qualified_table_name(location)?), + ( + &request + .optional("read_columns")? + .unwrap_or_else(|| py.None().into_bound(py)), + &request + .optional("render_add_columns")? + .unwrap_or_else(|| py.None().into_bound(py)), + writer.render_framework_type, + ), + ) + }, + describe, + ) } fn execute_sql<'py>(writer: &Writer<'_, 'py>, sql: &str) -> PyResult> { @@ -156,22 +186,14 @@ fn error_details<'py>(py: Python<'py>, error: &PyErr) -> PyResult( - writer: &Writer<'_, 'py>, - sqls: &[String], +/// Retry a complete warehouse operation after transient exceptions within the write budget. +fn retry_write<'py>( + py: Python<'py>, + write: impl Fn() -> PyResult<()>, describe: &dyn Fn(&PyErr) -> PyResult>, ) -> PyResult<()> { - let py: Python<'py> = writer.execute.py(); for attempt in 0..WRITE_ATTEMPTS { - let mut failure: Option = None; - for sql in sqls { - if let Err(error) = execute_sql(writer, sql) { - failure = Some(error); - break; - } - } - let Some(error) = failure else { + let Err(error) = write() else { return Ok(()); }; if attempt + 1 == WRITE_ATTEMPTS || !error.is_instance_of::(py) { @@ -376,16 +398,16 @@ fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult return Err(error), } } - let mut sqls: Vec = setup_sqls(&location, &writer)?; - sqls.push(insert); - execute_with_retry(&writer, &sqls, &|error| { + let describe = |error: &PyErr| { let mut details: Vec> = vec![ native_field(&fingerprint, "node_type")?, native_field(&fingerprint, "node_name")?, ]; details.extend(error_details(py, error)?); Ok((SINGLE_RETRY_MESSAGE, details)) - })?; + }; + setup(&location, &writer, &request, &describe)?; + retry_write(py, || execute_sql(&writer, &insert).map(|_| ()), &describe)?; Ok(true) } @@ -420,8 +442,7 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let on_progress: Option> = present(&request, "on_progress")?; let parts: WriteParts<'py> = WriteParts::read(&request)?; let (location, writer) = (parts.location(), parts.writer()); - let setup: Vec = setup_sqls(&location, &writer)?; - execute_with_retry(&writer, &setup, &|error| { + setup(&location, &writer, &request, &|error| { batch_warning(&fingerprints, error) })?; let batches = fingerprint_insert_batches( @@ -434,9 +455,11 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> let mut completed: usize = 0; for (batch, insert_sql) in batches { let members: Vec> = batch.iter().collect(); - execute_with_retry(&writer, &[insert_sql], &|error| { - batch_warning(&members, error) - })?; + retry_write( + py, + || execute_sql(&writer, &insert_sql).map(|_| ()), + &|error| batch_warning(&members, error), + )?; completed += members.len(); if let Some(progress) = &on_progress { let keywords: Bound<'py, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index ab91f568ae..598be15e62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -177,6 +177,11 @@ fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { "render_create_index_sqls", native_field(adapter, "render_create_source_freshness_index_sqls")?, ), + ("read_columns", native_field(adapter, "get_columns")?), + ( + "render_add_columns", + native_field(adapter, "render_add_columns")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 292bada80f..4a004cbbda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -4,7 +4,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; -use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES}; +use sqlbuild_state::freshness::constants::{ + REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, + STRING_STATE_VALUE_TYPE, +}; use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; @@ -13,6 +16,7 @@ use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::freshness::support::{ INPUT_ERROR, exception_text, freshness_error, optional_str, presumed_utc_datetime, text, }; +use crate::bindings::_helpers::native_host::state_column_evolution::ensure_event_id_column; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; @@ -82,12 +86,20 @@ fn create_table_sql<'py>( ) -> PyResult> { let py: Python<'py> = database.py(); let qualified_name: String = qualified_table_name(render_qualified_name, (database, schema))?; - let columns: Bound<'py, PyAny> = - PyTuple::new(py, SOURCE_FRESHNESS_COLUMN_TYPES.map(|(column, _)| column))?.into_any(); + let columns: Bound<'py, PyAny> = PyTuple::new( + py, + SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .chain([SOURCE_FRESHNESS_EVENT_ID]) + .collect::>(), + )? + .into_any(); let column_types: Bound<'py, PyDict> = PyDict::new(py); for (column, declared) in SOURCE_FRESHNESS_COLUMN_TYPES { column_types.set_item(column, declared)?; } + column_types.set_item(SOURCE_FRESHNESS_EVENT_ID, STRING_STATE_VALUE_TYPE)?; let column_types: Bound<'py, PyAny> = column_types.into_any(); let required: Bound<'py, PyAny> = PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(); let sql: String = state_table_create_sql( @@ -180,7 +192,7 @@ freshness table to regenerate source freshness state." ) } -fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = row.py(); let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { @@ -306,6 +318,20 @@ pub(crate) fn write_source_freshness_records<'py>( call_with(&render_create, &location())? }; execute_sql((execute, connection), create_sql)?; + let qualified = qualified_table_name( + &native_field(renderers, "render_qualified_name")?, + (database, schema), + )?; + ensure_event_id_column( + (connection, execute), + (database, schema), + (SOURCE_FRESHNESS_TABLE_NAME, &qualified), + ( + &native_field(renderers, "read_columns")?, + &native_field(renderers, "render_add_columns")?, + &native_field(renderers, "render_framework_type")?, + ), + )?; let render_indexes: Bound<'py, PyAny> = native_field(renderers, "render_create_index_sqls")?; if !render_indexes.is_none() { for index_sql in call_with(&render_indexes, &location())?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs new file mode 100644 index 0000000000..84031a99e7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -0,0 +1,299 @@ +//! Compile in the location interpreter and publish data through its scoped HTTP credential. + +use std::collections::BTreeMap; +use std::path::Path; +use std::process::{Command, Stdio}; + +use chrono::{DateTime, Utc}; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::git_revisions::main::head_commit::head_commit; +use sqlbuild_orchestration::declarations::models::{ + Schedule, TriggerDeclaration, TriggerDefinition, +}; +use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; + +use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; +use crate::bindings::_helpers::compiled_project::snapshot::{ + NativeOrchestrationSnapshot, build_orchestration_snapshot, +}; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; +use crate::bindings::_helpers::location_command::runtime::run_runtime; +use crate::bindings::_helpers::location_command::worker::execute_worker; +use crate::bindings::_helpers::native_host::host::write_console; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; +use crate::bindings::_helpers::recall::session::InvocationRecall; +use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::RecordId; + +fn status(py: Python<'_>, message: &str) -> PyResult<()> { + write_console(py, "stderr", &format!("{message}\n")) +} + +/// Git is optional in deployed images; absent metadata is not fabricated from the last publication. +fn metadata( + py: Python<'_>, + arguments: &KeywordArguments<'_>, + directory: &Path, +) -> PyResult { + let location: String = arguments.value("location_id")?.extract()?; + let location_id = location + .parse() + .map_err(|_| PyValueError::new_err("location ID must be a UUID"))?; + let target: String = arguments.value("selected_target")?.extract()?; + let action = arguments.value("action")?.extract::>()?; + let generation = arguments + .value("generation")? + .extract::>()? + .or_else(|| (action.as_deref() == Some(EXECUTE_ACTION)).then_some(1)) + .ok_or_else(|| PyValueError::new_err("publish requires --generation"))?; + if target.trim().is_empty() || generation < 1 { + return Err(PyValueError::new_err( + "target must not be blank and generation must be positive", + )); + } + let (git_revision, git_dirty, git_author) = py.detach(|| -> PyResult<_> { + let revision = head_commit(directory); + let mut dirty = false; + let mut author = None; + if revision.is_some() { + for (is_dirty, options) in [ + (false, vec!["log", "-1", "--format=%an", "HEAD"]), + ( + true, + vec!["status", "--porcelain", "--untracked-files=normal"], + ), + ] { + let output = Command::new("git") + .current_dir(directory) + .args(options) + .stdin(Stdio::null()) + .output()?; + if !output.status.success() { + return Err(PyRuntimeError::new_err( + "could not read location Git metadata", + )); + } + let text = String::from_utf8_lossy(&output.stdout); + if is_dirty { + dirty = !text.trim().is_empty(); + } else { + author = Some(text.trim().to_owned()); + } + } + } + Ok((revision, dirty, author)) + })?; + Ok(SnapshotInput { + location_id, + target, + generation, + git_revision, + git_dirty, + git_author, + image_id: arguments.value("image_id")?.extract()?, + environment_hash: String::new(), + sqb_version: String::new(), + nodes: Default::default(), + documents: Default::default(), + }) +} + +fn compile_location<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + input: &SnapshotInput, +) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>)> { + status(py, "Discovering location project...")?; + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Discovered, + )?; + status(py, "Location project discovered. Compiling snapshot...")?; + let options = new_record_with( + py, + RecordId::CompilePipelineOptions, + &[("selected_target", arguments.value("selected_target")?)], + )?; + let compiled = compiled_phase((&discovered, &adapter), &options, &py.None().into_bound(py))?; + let project = field(&compiled, "project")?; + let view = project.cast::()?.borrow(); + let snapshot = build_orchestration_snapshot( + view, + &serde_json::to_string(input).map_err(|error| PyValueError::new_err(error.to_string()))?, + project_dir.extract()?, + arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action == EXECUTE_ACTION) + .then(Vec::new), + )?; + status(py, "Location snapshot compiled.")?; + Ok((snapshot, discovered)) +} + +pub(super) fn location_arguments<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + KeywordArguments::read( + py, + "run_location_command", + options, + ( + &[ + "project_dir", + "location_id", + "selected_target", + "generation", + "server_url", + ], + &["image_id", "action", "run_id", "claim_id"], + ), + ) +} + +pub(super) fn publish_location( + py: Python<'_>, + arguments: &KeywordArguments<'_>, +) -> PyResult<(SnapshotRecord, Option>)> { + if arguments + .value("action")? + .extract::>()? + .is_some_and(|action| action != PUBLISH_ACTION) + { + return Err(PyValueError::new_err( + "location serve must run through the native sqb executable", + )); + } + let bearer = std::env::var("SQLBUILD_LOCATION_TOKEN").map_err(|_| { + PyValueError::new_err("SQLBUILD_LOCATION_TOKEN must contain a UTF-8 location credential") + })?; + if bearer.trim().is_empty() { + return Err(PyValueError::new_err( + "SQLBUILD_LOCATION_TOKEN must not be blank", + )); + } + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, arguments, &directory)?; + let (snapshot, _) = compile_location(py, arguments, &input)?; + let triggers: BTreeMap = serde_json::from_value( + snapshot + .input + .documents + .get("triggers") + .cloned() + .unwrap_or_else(|| serde_json::json!({})), + ) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let refresh_at = triggers + .values() + .filter_map(|trigger| match trigger { + TriggerDeclaration::Schedule(TriggerDefinition { + schedule: Schedule::Calendar { refresh_at, .. }, + .. + }) => Some(*refresh_at), + TriggerDeclaration::Schedule(_) | TriggerDeclaration::Sensor(_) => None, + }) + .min(); + status(py, "Publishing location snapshot...")?; + let record = snapshot.publication( + py, + &arguments.value("server_url")?.extract::()?, + &bearer, + )?.map_err(|error| { + let message = if matches!(error, sqlbuild_orchestration::api::errors::ClientError::Transport(_)) { + format!("Publication outcome is unknown; generation {} may already be published. Check the current snapshot before retrying: {error}", input.generation) + } else { + error.to_string() + }; + PyValueError::new_err(message) + })?; + status(py, "Location snapshot published.")?; + Ok((record, refresh_at)) +} + +/// One-shot publication and the supervisor's private stdio runtime share the same compiler path. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn run_location_command<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + run_location(py, options, None) +} + +/// CLI dispatch retains the verified worker through invocation history delivery. +pub(crate) fn run_location<'py>( + py: Python<'py>, + options: Option>, + recall: Option<&InvocationRecall>, +) -> PyResult { + let arguments = location_arguments(py, options.clone())?; + if arguments + .value("action")? + .extract::>()? + .as_deref() + == Some("execute") + { + let directory = arguments + .value("project_dir")? + .extract::()?; + let input = metadata(py, &arguments, &directory)?; + let bearer = std::env::var("SQLBUILD_RUN_TOKEN") + .map_err(|_| PyValueError::new_err("execute requires SQLBUILD_RUN_TOKEN"))?; + let identity = ( + arguments.value("run_id")?.extract::()?, + arguments.value("claim_id")?.extract::()?, + ); + let (snapshot, discovered) = compile_location(py, &arguments, &input)?; + return execute_worker( + py, + (&snapshot, discovered), + &arguments, + (&bearer, identity, recall), + ); + } + if arguments + .value("action")? + .extract::>()? + .as_deref() + == Some("runtime") + { + return run_runtime(py, options.unwrap_or_else(|| PyDict::new(py))); + } + let publication = (|| { + let (record, _) = publish_location(py, &arguments)?; + let output = serde_json::to_string(&record) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + write_console(py, "stdout", &format!("{output}\n")).map_err(|error| { + PyValueError::new_err(format!( + "Snapshot {} was published, but result delivery failed: {error}", + record.snapshot_id + )) + })?; + Ok(0) + })(); + if publication.is_err() { + status(py, "Location command failed.")?; + } + publication +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_location_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs new file mode 100644 index 0000000000..ffee78c649 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -0,0 +1,5 @@ +/// Location actions accepted by the interpreter-side one-shot publisher. +pub(crate) const PUBLISH_ACTION: &str = "publish"; + +/// Workers verify the pinned nodes and environment without reevaluating declarations. +pub(crate) const EXECUTE_ACTION: &str = "execute"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs new file mode 100644 index 0000000000..63f708bfc2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod command; +mod constants; +mod runtime; +mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs new file mode 100644 index 0000000000..bc75c07fdd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -0,0 +1,108 @@ +//! A location keeps its interpreter warm; only the supervisor writes generation requests. + +use std::io::{self, Write}; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseHistoryRead; + +use crate::bindings::_helpers::boundary::python_calls::within; +use crate::bindings::_helpers::location_command::command::{location_arguments, publish_location}; +use crate::bindings::_helpers::native_host::host::{redirect_stdout_to_stderr, write_console}; +use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; +use crate::bindings::_helpers::native_host::warehouse_metadata::read_history; + +pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { + options.set_item("action", "publish")?; + let redirect = redirect_stdout_to_stderr(py)?; + within(&redirect, |_| { + loop { + let line = py.detach(|| -> io::Result> { + let mut line = String::new(); + Ok((io::stdin().read_line(&mut line)? != 0).then_some(line)) + })?; + let Some(line) = line else { + break; + }; + let request: serde_json::Value = serde_json::from_str(&line).map_err(|_| { + PyValueError::new_err( + "runtime expects a JSON generation or sensor evaluation per line", + ) + })?; + let output = if let Some(generation) = request.as_i64() { + options.set_item("generation", generation)?; + let arguments = location_arguments(py, Some(options.clone()))?; + let record = match publish_location(py, &arguments) { + Ok(record) => Some(record), + Err(error) => { + write_console( + py, + "stderr", + &format!("Location compilation/publication failed: {error}\n"), + )?; + None + } + }; + serde_json::to_string(&record) + } else if request.get("source_freshness").is_some() + || request.get("fingerprints").is_some() + { + let metadata: WarehouseHistoryRead = + serde_json::from_value(request).map_err(|_| { + PyValueError::new_err("runtime received an invalid metadata read") + })?; + let arguments = location_arguments(py, Some(options.clone()))?; + let page = match read_history(py, &arguments, &metadata) { + Ok(page) => Some(page), + Err(_) => { + write_console( + py, + "stderr", + "Warehouse metadata read failed. No page delivered.\n", + )?; + None + } + }; + serde_json::to_string(&page) + } else { + let evaluation: SensorEvaluationRequest = + serde_json::from_value(request).map_err(|_| { + PyValueError::new_err("runtime received an invalid sensor evaluation") + })?; + let arguments = location_arguments(py, Some(options.clone()))?; + if arguments.value("location_id")?.extract::()? + != evaluation.location_id.to_string() + || arguments.value("selected_target")?.extract::()? != evaluation.target + { + return Err(PyValueError::new_err( + "sensor work belongs to another location or target", + )); + } + let directory = arguments + .value("project_dir")? + .extract::()?; + let outcome = match evaluate_sensor(py, &directory, &evaluation) { + Ok(outcome) => outcome, + Err(_) => { + write_console( + py, + "stderr", + "Custom sensor evaluation failed. No cursor or run requests committed.\n", + )?; + CustomSensorOutcome::Failed + } + }; + serde_json::to_string(&outcome) + }; + let output = output.map_err(|error| PyValueError::new_err(error.to_string()))?; + py.detach(|| -> io::Result<()> { + let mut stdout = io::stdout().lock(); + writeln!(stdout, "{output}")?; + stdout.flush() + })?; + } + Ok(0) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs new file mode 100644 index 0000000000..b7876f07f4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -0,0 +1,177 @@ +//! Execute only the API's frozen request after compiling and verifying the unmodified baseline. + +use std::sync::{Arc, mpsc}; +use std::time::Duration; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; +use sqlbuild_cli::dispatch::models::Dispatch; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_orchestration::runs::models::{RunOutcome, RunWindow}; + +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, within}; +use crate::bindings::_helpers::build_command::command::run_build; +use crate::bindings::_helpers::cli::argv::parse_cli_invocation; +use crate::bindings::_helpers::cli::dispatch::{Invocation, command_facts, warehouse_scope}; +use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; +use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::native_host::host::{entry, write_console}; +use crate::bindings::_helpers::recall::session::InvocationRecall; +use crate::bindings::_helpers::records::fields::{field, replaced}; + +fn build_request<'py>( + py: Python<'py>, + discovered: Bound<'py, PyAny>, + arguments: &KeywordArguments<'py>, + worker: &Bound<'py, NativeOrchestrationWorker>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let execution = worker.get().execution(py)?; + let frozen = execution.claim.request; + let mut settings = frozen.settings; + if let Some(window) = frozen.window { + for key in [ + "start_cursor", + "end_cursor", + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", + ] { + settings.remove(key); + } + let (kind, start, end) = match window { + RunWindow::Timestamp { start, end } => ("ts", start.to_rfc3339(), end.to_rfc3339()), + RunWindow::Integer { start, end } => ("int", start.to_string(), end.to_string()), + }; + settings.insert(format!("start_cursor_{kind}"), serde_json::json!(start)); + settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); + } + let mut argv: Vec> = std::iter::once("build".to_owned()) + .chain(job_settings_arguments(&settings).map_err(PyValueError::new_err)?) + .map(|argument| PyString::new(py, &argument)) + .collect(); + argv.extend([ + PyString::new(py, "--project-dir"), + arguments.value("project_dir")?.str()?, + PyString::new(py, "--target"), + PyString::new(py, &frozen.target), + PyString::new(py, "--no-color"), + PyString::new(py, "--json"), + ]); + for key in &frozen.selection { + if execution.skipped_nodes.contains_key(key) { + continue; + } + let (_, name) = key + .split_once(':') + .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; + argv.extend([PyString::new(py, "--select"), PyString::new(py, name)]); + } + let parsed = parse_cli_invocation(py, argv, false)?; + let args = field(&parsed, "args")?; + if args.is_none() { + return Err(PyValueError::new_err("invalid frozen build settings")); + } + let Dispatch::Run(route, None) = command_dispatch(&command_facts(&args)?) else { + return Err(PyValueError::new_err( + "frozen settings must dispatch a build", + )); + }; + let request = Invocation::new(&args)?.value(route.arguments[0].source)?; + Ok(( + replaced(&request, &[("discovered_inputs", discovered)])?, + warehouse_scope(&args)?, + )) +} + +/// The heartbeat is native; only driver cancellation crosses the executor host boundary. +fn run_cancellable( + py: Python<'_>, + worker: &Arc, + request: &Bound<'_, PyAny>, +) -> PyResult { + let state = Arc::clone(worker); + let (stop, stopping) = mpsc::channel::<()>(); + let thread = std::thread::Builder::new() + .name("sqb-worker-cancel".into()) + .spawn(move || { + while matches!( + stopping.recv_timeout(Duration::from_millis(100)), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + let _ = Python::attach(|py| -> PyResult<()> { + if state.worker.bind(py).get().cancel_requested(py) { + call_with(&entry(py, ExecutorHost::CancelRunningStatements)?, &[])?; + } + Ok(()) + }); + } + }) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = run_build(py, request, Some(Arc::clone(worker))); + drop(stop); + py.detach(|| thread.join()) + .map_err(|_| PyRuntimeError::new_err("worker cancellation thread failed"))?; + result +} + +pub(super) fn execute_worker<'py>( + py: Python<'py>, + (snapshot, discovered): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>), + arguments: &KeywordArguments<'py>, + (bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>), +) -> PyResult { + write_console(py, "stderr", "Verifying frozen run inputs...\n")?; + let owner = Bound::new( + py, + NativeOrchestrationWorker::start( + snapshot, + py, + (&arguments.value("server_url")?.extract::()?, bearer), + identity, + ) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + )?; + let worker = WorkerBuild::new(&owner)?; + if let Some(recall) = recall { + *recall + .worker + .lock() + .map_err(|_| PyRuntimeError::new_err("worker completion lock is unavailable"))? = + Some(Arc::clone(&worker)); + } + let result = (|| { + if !worker.has_work() { + write_console( + py, + "stderr", + "Run inputs verified. All selected nodes are blocked by upstream failures.\n", + )?; + return Ok(0); + } + let (request, scope) = build_request(py, discovered, arguments, &owner)?; + write_console( + py, + "stderr", + "Run inputs verified. Executing frozen selection...\n", + )?; + within(&scope, |_| run_cancellable(py, &worker, &request)) + })(); + let code = result.as_ref().copied().unwrap_or(1); + let outcome = if recall.is_some() { + worker.outcome(py, code)? + } else { + worker.finish(py, code)? + }; + result.map(|code| { + if matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { + code + } else { + code.max(1) + } + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 457ca748d9..732a57220e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -59,6 +59,7 @@ pub(crate) mod janitor; pub(crate) mod lifecycle_events; pub(crate) mod lineage; pub(crate) mod load; +pub(crate) mod location_command; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod microbatches; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs new file mode 100644 index 0000000000..035588e5e9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -0,0 +1,9 @@ +//! Authoring parameters reserved by the code-location host. + +/// The calendar's evaluation window is supplied by the host, not a provider. +pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; + +pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; + +/// The sensor context is supplied by the host, not a provider. +pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index c2904c48dc..babcc38c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -65,6 +65,21 @@ pub(crate) fn entry<'py>(py: Python<'py>, id: impl Into) -> PyResult, stream: &str, text: &str) -> PyResult<()> { + let stream = py.import("sys")?.getattr(stream)?; + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) +} + +/// Keep user prints off machine-readable stdout while running project code. +pub(crate) fn redirect_stdout_to_stderr(py: Python<'_>) -> PyResult> { + py.import("contextlib")? + .getattr("redirect_stdout")? + .call1((py.import("sys")?.getattr("stderr")?,)) +} + /// `log_debug_event(logger=logging.getLogger(logger), message=message)`. pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { let kwargs: Bound<'_, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 8f9ded74e2..248bf8cc95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,10 +1,17 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. pub(crate) mod connections; +pub(crate) mod constants; pub(crate) mod header_projection; pub(crate) mod host; pub(crate) mod model_lookup; +pub(crate) mod orchestration_declarations; +pub(crate) mod orchestration_environment; +pub(crate) mod orchestration_sensors; +pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod results; pub(crate) mod sqlite_connection; pub(crate) mod sqlite_cursor; +pub(crate) mod state_column_evolution; +pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs new file mode 100644 index 0000000000..12796bb6fe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -0,0 +1,221 @@ +//! The public authoring records enter native compilation here, never scheduler execution. + +use std::collections::{BTreeMap, HashSet}; +use std::path::Path; + +use pyo3::FromPyObject; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use serde_json::Value; +use sqlbuild_host::entries::models::UserCodeHost; + +use crate::bindings::_helpers::native_host::constants::RUN_EVENT_CONDITION; +use crate::bindings::_helpers::native_host::host::entry; + +#[derive(FromPyObject)] +pub(crate) struct AuthoredJob { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) select: String, + pub(crate) exclude: String, + pub(crate) overlap: String, + pub(crate) on_failure: String, + #[pyo3(from_py_with = authored_setting)] + pub(crate) settings: Value, + #[pyo3(from_py_with = authored_setting)] + pub(crate) concurrency: Value, + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) targets: Vec, +} + +#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct AuthoredJobReference { + pub(crate) name: String, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredGroup { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) jobs: Vec, + pub(crate) after: BTreeMap>, + pub(crate) overlap: String, + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) targets: Vec, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredSensorCondition { + pub(crate) kind: String, + pub(crate) kinds: Vec, + #[pyo3(default)] + pub(crate) job: Option, + #[pyo3(default)] + pub(crate) executed_successfully: Option>, + #[pyo3(default)] + pub(crate) select: Option, +} + +#[derive(FromPyObject)] +pub(crate) struct AuthoredSensor { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) when: Option, + pub(crate) function: Option>, + pub(crate) run: AuthoredJobReference, + #[pyo3(from_py_with = authored_setting)] + pub(crate) minimum_interval_seconds: Value, + pub(crate) default_status: String, + pub(crate) targets: Vec, +} + +pub(crate) enum AuthoredDeclaration { + Job(AuthoredJob), + Group(AuthoredGroup), + Calendar(Py), + Sensor(AuthoredSensor), +} + +/// JSON authoring values cross once; the interpreter rejects circular user containers safely. +pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult { + let text: String = value + .py() + .import("json")? + .call_method1("dumps", (value,))? + .extract()?; + serde_json::from_str(&text).map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// Only module execution crosses into Python; discovery and collection stay native. +pub(crate) fn module_jobs<'py>( + py: Python<'py>, + file_path: &Path, + project_dir: &Path, +) -> PyResult>> { + let options = PyDict::new(py); + options.set_item("file_path", file_path)?; + options.set_item("project_dir", project_dir)?; + let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; + let job_class = py.import("sqlbuild.orchestration")?.getattr("Job")?; + let group_class = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; + let calendar_class = py + .import("sqlbuild.orchestration")? + .getattr("CalendarSchedule")?; + let sensor_class = py.import("sqlbuild.orchestration")?.getattr("Sensor")?; + let attributes = module.getattr("__dict__")?.cast_into::()?; + let mut pending: Vec> = attributes.values().iter().collect(); + let mut seen: HashSet = HashSet::new(); + let mut jobs: Vec> = Vec::new(); + while let Some(value) = pending.pop() { + if !seen.insert(value.as_ptr() as usize) { + continue; + } + if value.is_instance(&job_class)? + || value.is_instance(&group_class)? + || value.is_instance(&calendar_class)? + || value.is_instance(&sensor_class)? + { + jobs.push(value); + } else if value.is_instance_of::() || value.is_instance_of::() { + pending.extend(value.try_iter()?.collect::>>()?); + } + } + Ok(jobs) +} + +pub(crate) fn collect_declarations<'py>( + py: Python<'py>, + declarations: Vec>, +) -> PyResult> { + let job = py.import("sqlbuild.orchestration")?.getattr("Job")?; + let group = py.import("sqlbuild.orchestration")?.getattr("JobGroup")?; + let calendar = py + .import("sqlbuild.orchestration")? + .getattr("CalendarSchedule")?; + let sensor = py.import("sqlbuild.orchestration")?.getattr("Sensor")?; + let mut roots: HashSet = HashSet::new(); + for declaration in &declarations { + if !roots.insert(declaration.as_ptr() as usize) { + if declaration.is_instance(&calendar)? { + return Err(PyValueError::new_err("duplicate calendar declaration")); + } + return Err(PyValueError::new_err(format!( + "duplicate job {}", + declaration.getattr("name")?.extract::()? + ))); + } + } + let mut pending = declarations; + let mut seen: HashSet = HashSet::new(); + let mut collected: Vec = Vec::new(); + while let Some(declaration) = pending.pop() { + if !seen.insert(declaration.as_ptr() as usize) { + continue; + } + let authored = if declaration.is_instance(&sensor)? { + let target = declaration.getattr("run")?; + if !target.is_instance(&job)? && !target.is_instance(&group)? { + return Err(PyTypeError::new_err( + "sensor run must be a Job or JobGroup object", + )); + } + pending.push(target); + let condition = declaration.getattr("when")?; + if !condition.is_none() + && condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION + { + let source = condition.getattr("job")?; + if !source.is_instance(&job)? && !source.is_instance(&group)? { + return Err(PyTypeError::new_err( + "run_event job must be a Job or JobGroup object", + )); + } + pending.push(source); + } + AuthoredDeclaration::Sensor(declaration.extract()?) + } else if declaration.is_instance(&calendar)? { + let target = declaration.getattr("job")?; + if !target.is_instance(&job)? && !target.is_instance(&group)? { + return Err(PyTypeError::new_err( + "calendar job must be a Job or JobGroup object", + )); + } + pending.push(target); + AuthoredDeclaration::Calendar(declaration.unbind()) + } else if declaration.is_instance(&group)? { + for member in declaration.getattr("jobs")?.try_iter()? { + let member = member?; + if !member.is_instance(&job)? { + return Err(PyTypeError::new_err("group members must be Job objects")); + } + pending.push(member); + } + for (member, dependencies) in declaration.getattr("after")?.cast::()?.iter() { + if !member.is_instance(&job)? { + return Err(PyTypeError::new_err( + "group dependencies must be Job objects", + )); + } + for dependency in dependencies.try_iter()? { + if !dependency?.is_instance(&job)? { + return Err(PyTypeError::new_err( + "group dependencies must be Job objects", + )); + } + } + } + AuthoredDeclaration::Group(declaration.extract()?) + } else if !declaration.is_instance(&job)? { + return Err(PyTypeError::new_err( + "snapshot declarations must be Job, JobGroup, CalendarSchedule or Sensor objects", + )); + } else { + AuthoredDeclaration::Job(declaration.extract()?) + }; + collected.push(authored); + } + Ok(collected) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs new file mode 100644 index 0000000000..26cfa2bf67 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_environment.rs @@ -0,0 +1,108 @@ +//! Read the location interpreter's installed dependencies, never its credentials or paths. + +use std::path::Path; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sha2::{Digest, Sha256}; +use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::ProjectTree; + +/// A lockfile binds declared dependencies; without one, bind installed distribution versions. +pub(crate) fn environment_hash(py: Python<'_>, project_dir: &Path) -> PyResult { + let mut hash = Sha256::new(); + let version = py.version_info(); + hash.update(format!( + "sqlbuild:{}\npython:{}.{}.{}{}\n", + env!("CARGO_PKG_VERSION"), + version.major, + version.minor, + version.patch, + version.suffix.unwrap_or_default(), + )); + match py.detach(|| std::fs::read(project_dir.join("uv.lock"))) { + Ok(lockfile) => { + hash.update(b"uv.lock\n"); + hash.update(lockfile); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let mut packages: Vec<(String, String)> = Vec::new(); + for distribution in py + .import("importlib.metadata")? + .call_method0("distributions")? + .try_iter()? + { + let distribution = distribution?; + let name: String = distribution + .getattr("metadata")? + .get_item("Name")? + .extract()?; + let version: String = distribution.getattr("version")?.extract()?; + packages.push((name.replace(['_', '.'], "-").to_ascii_lowercase(), version)); + } + packages.sort(); + hash.update(b"installed-packages\n"); + hash.update( + serde_json::to_vec(&packages) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?, + ); + } + Err(error) => return Err(error.into()), + } + Ok(format!("{:x}", hash.finalize())) +} + +/// Hash declaration/helper sources once per compilation, never once per sensor. +pub(crate) fn orchestration_code_hash(project_dir: &Path) -> PyResult { + let tree = ProjectTree::new(project_dir); + let files = rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { + !entry.is_dir && entry.name.ends_with(PYTHON_FILE_SUFFIX) + }) + .map_err(|error| { + pyo3::exceptions::PyValueError::new_err(format!( + "sensor source discovery failed: {error:?}" + )) + })?; + let mut hash = Sha256::new(); + for file in files { + let bytes = std::fs::read(tree.absolute(&file))?; + hash.update(file.len().to_le_bytes()); + hash.update(file.as_bytes()); + hash.update(bytes.len().to_le_bytes()); + hash.update(bytes); + } + Ok(format!("{:x}", hash.finalize())) +} + +/// Include the callable's code even for public in-memory declaration compilation. +pub(crate) fn callback_code_hash(function: &Bound<'_, PyAny>, sources: &str) -> PyResult { + let bytes: Vec = function + .py() + .import("marshal")? + .call_method1("dumps", (portable_code(function.getattr("__code__")?)?,))? + .extract()?; + let mut hash = Sha256::new(); + hash.update(sources.as_bytes()); + hash.update(bytes); + Ok(format!("{:x}", hash.finalize())) +} + +/// Code objects retain absolute import paths, including inside nested functions. +fn portable_code(code: Bound<'_, PyAny>) -> PyResult> { + let py = code.py(); + let code_type = py.import("types")?.getattr("CodeType")?; + let mut constants: Vec> = Vec::new(); + for value in code.getattr("co_consts")?.try_iter()? { + let value = value?; + constants.push(if value.is_instance(&code_type)? { + portable_code(value)? + } else { + value + }); + } + let changes = PyDict::new(py); + changes.set_item("co_filename", "")?; + changes.set_item("co_consts", PyTuple::new(py, constants)?)?; + code.call_method("replace", (), Some(&changes)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs new file mode 100644 index 0000000000..e6c05e8861 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -0,0 +1,132 @@ +//! User callbacks and provider lifetimes belong only to the isolated location interpreter. + +use std::path::Path; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::sensors::models::{ + CustomSensorOutcome, SensorCondition, SensorEvaluationRequest, +}; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; +use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; +use crate::bindings::_helpers::native_host::host::entry; +use crate::bindings::_helpers::native_host::orchestration_environment::{ + callback_code_hash, orchestration_code_hash, +}; + +pub(crate) fn evaluate_sensor( + py: Python<'_>, + project_dir: &Path, + evaluation: &SensorEvaluationRequest, +) -> PyResult { + let SensorCondition::Custom { + function: name, + code_hash, + } = &evaluation.definition.when + else { + return Err(PyValueError::new_err("custom sensor work required")); + }; + let public = py.import("sqlbuild.orchestration")?; + let sensor_class = public.getattr("Sensor")?; + let mut matches: Vec> = Vec::new(); + for declaration in discover_declarations(py, project_dir)? { + if declaration.is_instance(&sensor_class)? + && declaration.getattr("name")?.extract::()? == *name + { + matches.push(declaration); + } + } + let [sensor] = matches.as_slice() else { + return Err(PyValueError::new_err( + "custom sensor declaration must resolve uniquely", + )); + }; + let targets: Vec = sensor.getattr("targets")?.extract()?; + if !sensor.getattr("when")?.is_none() + || (!targets.is_empty() && !targets.contains(&evaluation.target)) + || sensor + .getattr("run")? + .getattr("name")? + .extract::()? + != evaluation.definition.job_key + { + return Err(PyValueError::new_err("custom sensor definition changed")); + } + let function = sensor.getattr("function")?; + let sources = py.detach(|| orchestration_code_hash(project_dir))?; + if callback_code_hash(&function, &sources)? != *code_hash { + return Err(PyValueError::new_err("custom sensor code changed")); + } + let fields = PyDict::new(py); + fields.set_item("cursor", &evaluation.cursor.custom)?; + fields.set_item("target", &evaluation.target)?; + fields.set_item("evaluation_id", evaluation.evaluation_id.to_string())?; + let context = public.getattr("SensorContext")?.call((), Some(&fields))?; + let providers = + discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + within(&session, |_| { + let arguments = PyDict::new(py); + arguments.set_item("ctx", &context)?; + for usage in provider_usages(&function, &by_name)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? != SENSOR_CONTEXT_PARAMETER { + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + } + let result = function.call((), Some(&arguments))?; + if result.is_instance(&public.getattr("SkipReason")?)? { + let reason: String = result.getattr("reason")?.extract()?; + if reason.trim().is_empty() { + return Err(PyValueError::new_err("SkipReason needs a nonempty reason")); + } + return Ok(CustomSensorOutcome::Skipped { + cursor: context.getattr("cursor")?.extract()?, + reason, + }); + } + let request_class = public.getattr("RunRequest")?; + let mut run_keys: Vec = Vec::new(); + for request in result + .try_iter()? + .take(sqlbuild_orchestration::sensors::constants::MAX_SENSOR_RUN_REQUESTS + 1) + { + let request = request?; + if !request.is_instance(&request_class)? { + return Err(PyValueError::new_err( + "custom sensors return RunRequest records or SkipReason", + )); + } + let key: String = request.getattr("run_key")?.extract()?; + if key.trim().is_empty() { + return Err(PyValueError::new_err("RunRequest needs a nonempty run_key")); + } + run_keys.push(key); + } + if run_keys.len() > sqlbuild_orchestration::sensors::constants::MAX_SENSOR_RUN_REQUESTS { + return Err(PyValueError::new_err( + "custom sensor exceeds the bounded run-request page", + )); + } + Ok(CustomSensorOutcome::Completed { + cursor: context.getattr("cursor")?.extract()?, + run_keys, + }) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs new file mode 100644 index 0000000000..256be90f0c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -0,0 +1,332 @@ +//! Authored trigger controls and bounded calendars evaluated only in the location interpreter. + +use std::collections::BTreeSet; +use std::path::Path; + +use chrono::{DateTime, TimeDelta, Timelike, Utc}; +use chrono_tz::Tz; +use pyo3::FromPyObject; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use serde_json::{Value, json}; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; +use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; +use sqlbuild_orchestration::sensors::models::SensorDefinition; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::native_host::constants::CALENDAR_WINDOW_PARAMETER; +use crate::bindings::_helpers::native_host::host::entry; +use crate::bindings::_helpers::native_host::orchestration_declarations::{ + AuthoredSensor, authored_setting, +}; +use crate::bindings::_helpers::native_host::orchestration_environment::{ + callback_code_hash, orchestration_code_hash, +}; + +#[derive(FromPyObject)] +struct AuthoredControls { + #[pyo3(from_py_with = authored_setting)] + starts_at: Value, + default_status: String, + catch_up: String, + #[pyo3(from_py_with = authored_setting)] + max_catch_up: Value, +} + +fn compile_trigger( + authored: &Bound<'_, PyAny>, + job_key: &str, + description: &str, + schedule: Schedule, +) -> PyResult { + let controls = authored.extract::()?; + let catch_up = if controls.max_catch_up.is_null() { + Value::String(controls.catch_up) + } else { + serde_json::json!({controls.catch_up: {"max_runs": controls.max_catch_up}}) + }; + Ok(TriggerDefinition { + description: description.into(), + job_key: job_key.into(), + schedule, + starts_at: serde_json::from_value(controls.starts_at) + .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, + default_status: serde_json::from_value(Value::String(controls.default_status)) + .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, + catch_up: serde_json::from_value(catch_up).map_err(|error| PyValueError::new_err(format!( + "catch_up: {error}; use 'latest' without a bound or 'all' with a positive max_catch_up" + )))?, + }) +} + +pub(crate) fn compile_cron( + authored: &Bound<'_, PyAny>, + name: &str, + description: &str, +) -> PyResult { + compile_trigger( + authored, + name, + description, + Schedule::Cron { + expression: authored.getattr("cron")?.extract()?, + timezone: authored.getattr("timezone")?.extract()?, + }, + ) +} + +fn horizon(value: &str) -> PyResult { + let invalid = || { + PyValueError::new_err( + "calendar horizon must be a positive fixed duration using d, h, m or s, within datetime range", + ) + }; + let digits = parse_duration(value).ok_or_else(invalid)?; + if [&digits.years, &digits.months] + .iter() + .any(|digits| !digits.trim_start_matches('0').is_empty()) + { + return Err(invalid()); + } + let seconds = [ + (digits.days, 86_400_i64), + (digits.hours, 3_600), + (digits.minutes, 60), + (digits.seconds, 1), + ] + .into_iter() + .try_fold(0_i64, |total, (digits, factor)| { + let digits = digits.parse::().map_err(|_| invalid())?; + let seconds = digits.checked_mul(factor).ok_or_else(invalid)?; + total.checked_add(seconds).ok_or_else(invalid) + })?; + TimeDelta::try_seconds(seconds).ok_or_else(invalid) +} + +fn calendar_times( + authored: &Bound<'_, PyAny>, + session: &Bound<'_, PyAny>, + providers: &Bound<'_, PyDict>, + start: DateTime, +) -> PyResult { + let py = authored.py(); + let timezone: String = authored.getattr("timezone")?.extract()?; + timezone + .parse::() + .map_err(|_| PyValueError::new_err("calendar timezone must be an IANA timezone"))?; + let duration = horizon(&authored.getattr("horizon")?.extract::()?)?; + let end = start + .checked_add_signed(duration) + .ok_or_else(|| PyValueError::new_err("calendar horizon is out of datetime range"))?; + let datetime = imported(py, "datetime", "datetime")?; + let zone = imported(py, "zoneinfo", "ZoneInfo")?.call1((timezone,))?; + let window = PyDict::new(py); + for (name, instant) in [("start", start), ("end", end)] { + window.set_item( + name, + datetime + .call_method1("fromisoformat", (instant.to_rfc3339(),))? + .call_method1("astimezone", (&zone,))?, + )?; + } + let window = py + .import("sqlbuild.orchestration")? + .getattr("CalendarWindow")? + .call((), Some(&window))?; + let function = authored.getattr("function")?; + let arguments = PyDict::new(py); + arguments.set_item(CALENDAR_WINDOW_PARAMETER, window)?; + for usage in provider_usages(&function, providers)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? == CALENDAR_WINDOW_PARAMETER { + continue; + } + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + let mut times: BTreeSet> = BTreeSet::new(); + for (index, time) in function + .call((), Some(&arguments))? + .try_iter()? + .take(MAX_TICKS_PER_BATCH + 1) + .enumerate() + { + if index == MAX_TICKS_PER_BATCH { + return Err(PyValueError::new_err(format!( + "calendar exceeds {MAX_TICKS_PER_BATCH} upcoming times" + ))); + } + let time = time?; + if !time.is_instance(&datetime)? { + return Err(PyValueError::new_err( + "calendar times must be timezone-aware datetimes", + )); + } + let time = + DateTime::parse_from_rfc3339(&time.call_method0("isoformat")?.extract::()?) + .map_err(|_| { + PyValueError::new_err("calendar times must be timezone-aware datetimes") + })? + .with_timezone(&Utc); + if time < start || time >= end { + return Err(PyValueError::new_err( + "calendar time lies outside its half-open evaluation window", + )); + } + times.insert(time); + } + Ok(Schedule::Calendar { + times, + refresh_at: start + duration / 2, + }) +} + +pub(crate) fn compile_calendars( + py: Python<'_>, + (project_dir, target): (&Path, &str), + authored: Vec>, + mut definitions: Definitions, +) -> PyResult { + let mut calendars: Vec> = Vec::new(); + for calendar in authored { + let targets = calendar + .bind(py) + .getattr("targets")? + .extract::>()?; + if targets.is_empty() || targets.iter().any(|value| value == target) { + calendars.push(calendar); + } + } + if calendars.is_empty() { + return Ok(definitions); + } + let providers = + discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + let now = Utc::now(); + let start = now + .with_nanosecond(now.timestamp_subsec_micros() * 1_000) + .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; + within(&session, |_| { + for calendar in calendars { + let calendar = calendar.bind(py); + let name: String = calendar + .getattr("function")? + .getattr("__name__")? + .extract()?; + let job_key: String = calendar.getattr("job")?.getattr("name")?.extract()?; + let description: String = calendar.getattr("description")?.extract()?; + if description.trim().is_empty() + || (!definitions.jobs.contains_key(&job_key) + && !definitions.groups.contains_key(&job_key)) + { + return Err(PyValueError::new_err(format!( + "calendar {name} needs a description and a published job" + ))); + } + if definitions.triggers.contains_key(&name) { + return Err(PyValueError::new_err(format!("duplicate trigger {name}"))); + } + let schedule = calendar_times(calendar, &session, &by_name, start)?; + let mut trigger = compile_trigger(calendar, &job_key, &description, schedule)?; + trigger.starts_at.get_or_insert(start); + definitions.triggers.insert(name, trigger); + } + Ok(definitions) + }) +} + +pub(crate) fn compile_sensors( + py: Python<'_>, + (project_dir, target, graph): (&Path, &str, &ProjectGraph), + authored: Vec, + mut definitions: Definitions, +) -> PyResult { + let sources = if authored.iter().any(|sensor| sensor.function.is_some()) { + orchestration_code_hash(project_dir)? + } else { + String::new() + }; + let mut names: BTreeSet = BTreeSet::new(); + for sensor in authored { + if !names.insert(sensor.name.clone()) || definitions.triggers.contains_key(&sensor.name) { + return Err(PyValueError::new_err(format!( + "duplicate trigger {}", + sensor.name + ))); + } + if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == target) { + continue; + } + let condition = match (sensor.when, sensor.function) { + (Some(when), None) => { + let mut condition = json!({"kind": when.kind, "kinds": when.kinds}); + if let Some(job) = when.job { + condition["job_key"] = json!(job.name); + } + if let Some(select) = when.select { + let selectors: Vec = + select.split_whitespace().map(str::to_owned).collect(); + condition["selection"] = sensor_nodes(graph, &selectors, "node_event")?; + } + if let Some(selectors) = when.executed_successfully { + condition["executed_successfully"] = + sensor_nodes(graph, &selectors, "executed_successfully")?; + } + condition + } + (None, Some(function)) => json!({"kind": "custom", "function": sensor.name, + "code_hash": callback_code_hash(function.bind(py), &sources)?}), + _ => { + return Err(PyValueError::new_err( + "sensor needs either a built-in condition or a decorated polling callback; callbacks with when are not supported yet", + )); + } + }; + let definition: SensorDefinition = serde_json::from_value(json!({ + "description": sensor.description, + "job_key": sensor.run.name, + "when": condition, + "minimum_interval_seconds": sensor.minimum_interval_seconds, + "default_status": sensor.default_status, + })) + .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; + definitions.sensors.insert(sensor.name, definition); + } + Ok(definitions) +} + +fn sensor_nodes(graph: &ProjectGraph, selectors: &[String], field: &str) -> PyResult { + if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + let nodes: BTreeSet = resolve_selectors(graph, selectors, &[]) + .map_err(|error| PyValueError::new_err(error.message))? + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if nodes.is_empty() { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + Ok(json!(nodes)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs new file mode 100644 index 0000000000..bfb47dcba2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/state_column_evolution.rs @@ -0,0 +1,85 @@ +//! Warehouse catalog and nullable-column DDL adapter edge for persisted event identity. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_host::entries::models::AdapterHost; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::field; + +/// Missing callbacks retain the existing renderer-only API for already-current/new schemas. +pub(crate) fn ensure_event_id_column<'py>( + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (table, qualified): (&str, &str), + (read_columns, render_add, render_type): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult<()> { + if read_columns.is_none() && render_add.is_none() { + return Ok(()); + } + if read_columns.is_none() || render_add.is_none() { + return Err(PyValueError::new_err( + "state column evolution requires both read_columns and render_add_columns", + )); + } + let py = connection.py(); + let scope = [ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", PyString::new(py, table).into_any()), + ]; + if event_id_present(read_columns, &scope)? { + return Ok(()); + } + let string_type = render_type.call1((field( + &host::entry(py, AdapterHost::FrameworkType)?, + "STRING", + )?,))?; + let column = call_with( + &host::entry(py, AdapterHost::ColumnInfo)?, + &[ + ("name", PyString::new(py, "event_id").into_any()), + ("type", string_type), + ], + )?; + let statements = call_with( + render_add, + &[ + ("destination", PyString::new(py, qualified).into_any()), + ("columns", PyTuple::new(py, [column])?.into_any()), + ], + )?; + for statement in statements.try_iter()? { + if let Err(error) = call_with( + execute, + &[("connection", connection.clone()), ("sql", statement?)], + ) && !event_id_present(read_columns, &scope).unwrap_or(false) + { + return Err(error); + } + } + Ok(()) +} + +/// Concurrent additions are harmless only once the authoritative catalog confirms the column. +fn event_id_present( + read_columns: &Bound<'_, PyAny>, + scope: &[(&str, Bound<'_, PyAny>)], +) -> PyResult { + for column in call_with(read_columns, scope)?.try_iter()? { + if field(&column?, "name")? + .extract::()? + .eq_ignore_ascii_case("event_id") + { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs new file mode 100644 index 0000000000..d07490aeb7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -0,0 +1,309 @@ +//! Read-only warehouse adapter edge; never compile declarations or write warehouse state. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use serde_json::{Value, json}; +use sqlbuild_analysis::fingerprint_storage::constants::{COLUMN_TIMESTAMP, FINGERPRINT_COLUMNS}; +use sqlbuild_config::command_connection::constants::{ + DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, +}; +use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_core::models::SqlValue; +use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; +use sqlbuild_orchestration::warehouse_metadata::models::{ + WarehouseHistoryPage, WarehouseHistoryRead, WarehouseHistoryRecord, +}; +use sqlbuild_state::freshness::constants::{ + OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, +}; +use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; +use sqlbuild_state::state_tables::models::HistoryRead; + +use crate::bindings::_helpers::authored_values::sql_values::{SqlValueClasses, sql_value}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; +use crate::bindings::_helpers::freshness::state_io::row_record; +use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::{construct::record_class, ids::RecordId}; + +pub(crate) fn read_history( + py: Python<'_>, + arguments: &KeywordArguments<'_>, + request: &WarehouseHistoryRead, +) -> PyResult { + let location = match request { + WarehouseHistoryRead::SourceFreshness(read) => &read.location, + WarehouseHistoryRead::Fingerprints(read) => &read.location, + }; + if location.schema.trim().is_empty() { + return Err(PyValueError::new_err( + "metadata state schema must not be blank", + )); + } + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Configuration, + )?; + let connection_config = project_connection_config( + &discovered, + (&project_dir, &arguments.value("selected_target")?), + (&py.None().into_bound(py), true), + )?; + let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; + if let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS + .iter() + .find(|(name, _)| *name == adapter_name) + { + let config = connection_config.cast::()?; + let path: Option = config + .get_item(path_key)? + .map(|path| path.extract()) + .transpose()?; + if path + .as_deref() + .is_some_and(|path| path != DUCKDB_MEMORY_DATABASE) + { + config.set_item("read_only", true)?; + } + if adapter_name == DUCKDB_ADAPTER + && let Some(attachments) = config.get_item("attach")? + { + for attachment in attachments.try_iter()? { + attachment?.set_item("read_only", true)?; + } + } + } + let connection = adapter.call_method1("connect", (connection_config,))?; + let read = read_page(py, (&adapter, &connection), request); + let closed = adapter.call_method1("close", (&connection,)); + let page = read?; + closed?; + Ok(page) +} + +fn read_page( + py: Python<'_>, + (adapter, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + request: &WarehouseHistoryRead, +) -> PyResult { + let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); + let (location, after, since, table, columns, timestamp) = match request { + WarehouseHistoryRead::SourceFreshness(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + SOURCE_FRESHNESS_TABLE_NAME, + freshness_columns.as_slice(), + OBSERVED_AT_COLUMN, + ), + WarehouseHistoryRead::Fingerprints(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + FINGERPRINT_TABLE_NAME, + FINGERPRINT_COLUMNS.as_slice(), + COLUMN_TIMESTAMP, + ), + }; + let database = location.database.as_ref().map_or_else( + || py.None().into_bound(py), + |value| PyString::new(py, value).into_any(), + ); + let schema = PyString::new(py, &location.schema).into_any(); + let name = PyString::new(py, table).into_any(); + let exists = call_with( + &adapter.getattr("relation_exists")?, + &[ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", name.clone()), + ], + )? + .is_truthy()?; + if !exists { + return Ok(WarehouseHistoryPage { + records: Vec::new(), + next_after: None, + }); + } + let catalog_columns = call_with( + &adapter.getattr("get_columns")?, + &[ + ("connection", connection.clone()), + ("database", database.clone()), + ("schema", schema.clone()), + ("name", name.clone()), + ], + )?; + let mut has_event_id = false; + for column in catalog_columns.try_iter()? { + has_event_id |= field(&column?, "name")? + .extract::()? + .eq_ignore_ascii_case("event_id"); + } + let mut literals = vec![None; columns.len() + 1]; + if let Some(after) = after { + let classes = SqlValueClasses::load(py)?; + for (output, value) in literals.iter_mut().zip(after) { + if let Some(value) = value { + let value = sql_value(&SqlValue::String(value.clone()), &classes)?; + *output = Some( + call_with( + &adapter.getattr("render_typed_scalar")?, + &[("value", value)], + )? + .extract()?, + ); + } + } + } + let qualified: String = call_with( + &adapter.getattr("render_qualified_name")?, + &[("database", database), ("schema", schema), ("name", name)], + )? + .extract()?; + let adapter_name = adapter.getattr("adapter_name")?.extract::()?; + let observed_since = since + .map(|since| { + let text = if adapter_name == SQLITE_ADAPTER { + since + .date_naive() + .pred_opt() + .unwrap_or(since.date_naive()) + .format("%Y-%m-%d") + .to_string() + } else { + since.format("%Y-%m-%dT%H:%M:%S+00:00").to_string() + }; + let classes = SqlValueClasses::load(py)?; + let value = sql_value(&SqlValue::String(text), &classes)?; + call_with( + &adapter.getattr("render_typed_scalar")?, + &[("value", value)], + )? + .extract::() + }) + .transpose()?; + let sql = read_history_sql( + &qualified, + after.map(|_| literals.as_slice()), + HistoryRead { + columns, + timestamp_column: timestamp, + observed_since: observed_since.as_deref(), + adapter: &adapter_name, + has_event_id, + limit: MAX_HISTORY_RECORDS, + }, + ) + .map_err(PyValueError::new_err)?; + let result = call_with( + &adapter.getattr("query")?, + &[ + ("connection", connection.clone()), + ("sql", PyString::new(py, &sql).into_any()), + ( + "limit", + pyo3::IntoPyObjectExt::into_bound_py_any(MAX_HISTORY_RECORDS, py)?, + ), + ], + )?; + let mut records: Vec = Vec::new(); + let mut next_after = None; + let mut read_count = 0; + for row in field(&result, "rows")?.try_iter()? { + read_count += 1; + let row = row?; + let record = decode_record(py, (request, &row, &qualified), (columns, timestamp))?; + let mut cursor = vec![None; columns.len() + 1]; + for (index, value) in cursor.iter_mut().enumerate() { + let raw = row.get_item(index)?; + *value = if raw.is_none() { + None + } else { + Some(raw.str()?.extract()?) + }; + } + let observed = match &record { + WarehouseHistoryRecord::SourceFreshness(record) => record.observed_at, + WarehouseHistoryRecord::Fingerprint(record) => record.ts, + }; + if since.is_none_or(|since| observed >= since) { + records.push(record); + } + next_after = Some(cursor); + } + if read_count < MAX_HISTORY_RECORDS { + next_after = None; + } + Ok(WarehouseHistoryPage { + records, + next_after, + }) +} + +fn decode_record( + py: Python<'_>, + (request, row, qualified): (&WarehouseHistoryRead, &Bound<'_, PyAny>, &str), + (columns, timestamp): (&[&str], &str), +) -> PyResult { + let (location, record) = match request { + WarehouseHistoryRead::SourceFreshness(read) => (&read.location, row_record(row)?), + WarehouseHistoryRead::Fingerprints(read) => ( + &read.location, + row_fingerprint( + &record_class(py, RecordId::Fingerprint)?, + &py.import("datetime")? + .getattr("datetime")? + .getattr("fromisoformat")?, + row, + qualified, + )?, + ), + }; + let mut value = json!({ + "state_database": location.database, + "state_schema": location.schema, + "event_id": row.get_item(columns.len())?.extract::>()?, + }); + for name in columns { + let name = match *name { + "definition_b64" => "definition", + "metadata_json_b64" => "metadata_json", + name => name, + }; + let item = field(&record, name)?; + value[name] = if item.is_none() { + Value::Null + } else if name == timestamp { + json!( + presumed_utc_datetime(&item)? + .call_method0("isoformat")? + .extract::()? + ) + } else { + json!(item.extract::()?) + }; + } + match request { + WarehouseHistoryRead::SourceFreshness(_) => { + serde_json::from_value(value).map(WarehouseHistoryRecord::SourceFreshness) + } + WarehouseHistoryRead::Fingerprints(_) => { + serde_json::from_value(value).map(WarehouseHistoryRecord::Fingerprint) + } + } + .map_err(|error| PyValueError::new_err(error.to_string())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 85714984a5..43b2fafcfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -36,6 +36,7 @@ use sqlbuild_recall::writer::models::{ DeliverySummary, RecallSession, SessionConfig, SessionStore, SetupOutcome, }; +use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::recall::clickhouse_store::{ @@ -56,6 +57,8 @@ use crate::bindings::_helpers::records::fields::field as native_field; /// What the invocation records: its session, when enabled, and the resolved target and adapter. #[derive(Clone, Debug, Default)] pub(crate) struct InvocationRecall { + /// The verified worker remains owned until Recall has drained and its claim can be released. + pub(crate) worker: Arc>>>, pub(crate) session: Option>, pub(crate) target: Option, pub(crate) adapter: Option, @@ -409,35 +412,38 @@ fn quarantined_notice(path: &Path) -> Option { fn started_session( py: Python<'_>, (args, identity, facts): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &CommandFacts), - (settings, project_dir): (&RecallSettings, &Path), + (settings, project_dir, hosted): (&RecallSettings, &Path, Option), ) -> PyResult>> { - let store: SessionStore = match &settings.store { - StoreLocation::Sqlite { path } => { - notify(py, quarantined_notice(path)); - if let Some(reason) = store_location_problem(path) { - notify(py, permission_notice((settings, project_dir), &reason)); + let store: SessionStore = match hosted { + Some(store) => store, + None => match &settings.store { + StoreLocation::Sqlite { path } => { + notify(py, quarantined_notice(path)); + if let Some(reason) = store_location_problem(path) { + notify(py, permission_notice((settings, project_dir), &reason)); + return Ok(None); + } + SessionStore::Sqlite(path.clone()) + } + StoreLocation::ClickHouse { + config, database, .. + } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { + location: ClickHouseLocation::new(config, database, settings.retention_days), + retention_notice: notice_due, + project_dir: project_dir.to_path_buf(), + information: Mutex::default(), + })), + StoreLocation::Unsupported { + connection, + adapter, + } => { + notify( + py, + unsupported_notice((settings, project_dir), (connection, adapter)), + ); return Ok(None); } - SessionStore::Sqlite(path.clone()) - } - StoreLocation::ClickHouse { - config, database, .. - } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { - location: ClickHouseLocation::new(config, database, settings.retention_days), - retention_notice: notice_due, - project_dir: project_dir.to_path_buf(), - information: Mutex::default(), - })), - StoreLocation::Unsupported { - connection, - adapter, - } => { - notify( - py, - unsupported_notice((settings, project_dir), (connection, adapter)), - ); - return Ok(None); - } + }, }; let Some(invocation_id) = text_value(&native_field(identity, "invocation_id")?)? else { return Ok(None); @@ -462,13 +468,44 @@ fn started_session( max_output_bytes: settings.max_output_bytes, heartbeat_interval: heartbeat_interval(), setup_budget: Duration::ZERO, - maintenance: true, + maintenance: !worker_invocation(args, facts)?, defer_context: true, audit_evidence: settings.audit_evidence, }); Ok(Some(Arc::new(session))) } +fn worker_invocation(args: &Bound<'_, PyAny>, facts: &CommandFacts) -> PyResult { + Ok(facts.command.as_deref() == Some("location") + && attribute(args, "location_command")? + .extract::>()? + .as_deref() + == Some("execute")) +} + +fn worker_store(args: &Bound<'_, PyAny>) -> PyResult { + let identity = ( + attribute(args, "location_run_id")? + .extract::()? + .parse() + .map_err(|_| PyValueError::new_err("run ID must be a UUID"))?, + attribute(args, "location_claim_id")? + .extract::()? + .parse() + .map_err(|_| PyValueError::new_err("claim ID must be a UUID"))?, + ); + let bearer = std::env::var("SQLBUILD_RUN_TOKEN") + .map_err(|_| PyValueError::new_err("execute requires SQLBUILD_RUN_TOKEN"))?; + let url = attribute(args, "location_server_url")?.extract::()?; + args.py().detach(|| { + sqlbuild_orchestration::worker::main::connect_worker::connect_worker( + &url, &bearer, identity, + ) + .map(|client| client.recall_store()) + .map_err(|error| PyValueError::new_err(error.to_string())) + }) +} + /// Begin recording when Recall is enabled, without waiting on the store. Only the write connection is resolved; a write connection without a read connection is kept for the command to fail with, and other unresolvable settings record nothing. pub(crate) fn start_invocation_recall( py: Python<'_>, @@ -482,7 +519,7 @@ pub(crate) fn start_invocation_recall( selected_target: text_value(&args.getattr("target")?)?, purpose: StorePurpose::Record, }; - let settings: RecallSettings = match resolve_recall_settings(&request) { + let mut settings: RecallSettings = match resolve_recall_settings(&request) { Ok(settings) => settings, Err(error) if error.invalid_configuration => { return Ok(InvocationRecall { @@ -492,14 +529,25 @@ pub(crate) fn start_invocation_recall( } Err(_unreachable_store) => return Ok(InvocationRecall::default()), }; + let hosted = if worker_invocation(args, facts)? { + settings.enabled = true; + Some(worker_store(args)?) + } else { + None + }; let session: Option> = if settings.enabled && settings.project_name.is_some() { - started_session(py, (args, identity, facts), (&settings, &project_dir))? + started_session( + py, + (args, identity, facts), + (&settings, &project_dir, hosted), + )? } else { None }; let audit_evidence: AuditEvidence = settings.audit_evidence; Ok(InvocationRecall { + worker: Arc::default(), session, target: settings.target.clone(), adapter: settings.project_adapter.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index d8e3df098d..3fdd402cc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -982,6 +982,7 @@ const CLI_ENTRYPOINT_HANDLERS: RecordSpec = RecordSpec { FieldSpec::new("run_refactor", FieldDefault::None), FieldSpec::new("run_recall", FieldDefault::None), FieldSpec::new("run_review", FieldDefault::None), + FieldSpec::new("run_location", FieldDefault::None), ], }; const COMMAND_WAREHOUSE_SCOPE: RecordSpec = RecordSpec { @@ -1366,6 +1367,8 @@ const SOURCE_FRESHNESS_RENDERERS: RecordSpec = RecordSpec { FieldSpec::required("render_insert_records_sql"), FieldSpec::new("render_create_table_sql", FieldDefault::None), FieldSpec::new("render_create_index_sqls", FieldDefault::None), + FieldSpec::new("read_columns", FieldDefault::None), + FieldSpec::new("render_add_columns", FieldDefault::None), ], }; const SOURCE_FRESHNESS_OBSERVATION: RecordSpec = RecordSpec { @@ -1560,6 +1563,7 @@ const CLI_COMMAND: EnumSpec = EnumSpec { ("DIFF", "diff"), ("REVIEW", "review"), ("DEBUG", "debug"), + ("LOCATION", "location"), ("LINEAGE", "lineage"), ("QUERY", "query"), ("COST", "cost"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index a59d0e0da2..349e4ebf1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -12,6 +12,7 @@ use sqlbuild_rules::configuration::errors::RulesConfigLoadError; use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; use sqlbuild_rules::models::RulesConfig; +use crate::bindings::_helpers::boundary::config_values::json_value_to_python; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record; @@ -64,31 +65,6 @@ fn option_scalar<'py>(py: Python<'py>, value: &Value) -> PyResult(py: Python<'py>, value: &Value) -> PyResult> { - match value { - Value::Array(items) => { - let values: Vec> = items - .iter() - .map(|item| option_item(py, item)) - .collect::>()?; - Ok(pyo3::types::PyList::new(py, values)?.into_any()) - } - Value::Object(entries) => { - let table: Bound<'py, PyDict> = PyDict::new(py); - for (key, item) in entries { - table.set_item(key, option_item(py, item)?)?; - } - Ok(table.into_any()) - } - Value::Null => Ok(py.None().into_bound(py)), - Value::Number(number) if number.as_i64().is_none() && number.as_u64().is_none() => { - number.as_f64().unwrap_or(f64::NAN).into_bound_py_any(py) - } - other => Ok(option_scalar(py, other)?.unwrap_or_else(|| py.None().into_bound(py))), - } -} - fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> { let result: Bound<'py, PyDict> = PyDict::new(py); for (code, options) in &config.rule_options { @@ -97,7 +73,7 @@ fn rule_options<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult> = items .iter() - .map(|item| option_item(py, item)) + .map(|item| json_value_to_python(py, item)) .collect::>()?; values.set_item(name, PyTuple::new(py, tuple)?)?; } else if let Some(scalar) = option_scalar(py, value)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs index 2a93fd2855..b02904a131 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -14,7 +14,9 @@ use sqlbuild_state::freshness::main::age_status::age_status; use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; -use sqlbuild_state::freshness::models::{AgeStatus, ComparableRecord, DataVersion}; +use sqlbuild_state::freshness::models::{ + AgeStatus, ComparableRecord, DataVersion, FreshnessEventIdentity, +}; use sqlbuild_state::freshness::types::TimestampParser; use crate::bindings::_helpers::boundary::python_calls::imported; @@ -31,6 +33,40 @@ const TIMESTAMP_KIND: &str = "timestamp"; const INTEGER_KIND: &str = "integer"; const STRING_KIND: &str = "string"; +/// Normalize timestamp offsets and precision before computing a persisted observation identity. +pub(crate) fn record_event_id(record: &Bound<'_, PyAny>) -> PyResult { + let observed_at = presumed_utc(&native_field(record, "observed_at")?)?; + let observed_at = format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}+00:00", + observed_at.date.year, + observed_at.date.month, + observed_at.date.day, + observed_at.hour, + observed_at.minute, + observed_at.second, + observed_at.microsecond, + ); + Ok(sqlbuild_state::freshness::main::event_id::event_id( + &FreshnessEventIdentity { + source_name: &native_field(record, "source_name")?.extract::()?, + target_database: native_field(record, "target_database")? + .extract::>()? + .as_deref(), + target_schema: native_field(record, "target_schema")? + .extract::>()? + .as_deref(), + target_name: native_field(record, "target_name")? + .extract::>()? + .as_deref(), + run_id: &native_field(record, "run_id")?.extract::()?, + strategy: &native_field(record, "strategy")?.extract::()?, + value_kind: &native_field(record, "value_kind")?.extract::()?, + data_version_hash: &native_field(record, "data_version_hash")?.extract::()?, + observed_at: &observed_at, + }, + )) +} + fn raised(py: Python<'_>, class: NativeAttr, message: String) -> PyErr { match registry::native(py, class).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs index 40a1d39bf6..9f7aca6b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -9,7 +9,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::constants::{ - SOURCE_FRESHNESS_COLUMN_TYPES, STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE, + SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, STRING_STATE_VALUE_TYPE, + TEXT_TIMESTAMP_STATE_VALUE_TYPE, }; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; @@ -20,6 +21,7 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::state_rows::freshness::record_event_id; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; /// Each state value type a state table may declare and the framework type that renders it. @@ -121,18 +123,20 @@ fn render_insert_source_freshness_records_sql<'py>( let mut rows: Vec> = Vec::new(); for record in records.try_iter()? { let record: Bound<'py, PyAny> = record?; - let row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES + let mut row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES .iter() .map(|(column, declared)| { render_state_sql_literal(py, &native_field(&record, column)?, declared, None) }) .collect::>()?; + row.push(format!("'{}'", record_event_id(&record)?)); rows.push(row); } - let names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES + let mut names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES .iter() .map(|(column, _)| *column) .collect(); + names.push(SOURCE_FRESHNESS_EVENT_ID); Ok(values_insert_sql( &table.extract::()?, &names, diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml index 5738b05067..8b54b4542e 100644 --- a/crates/sqlbuild-recall/Cargo.toml +++ b/crates/sqlbuild-recall/Cargo.toml @@ -9,11 +9,14 @@ rust-version.workspace = true [dependencies] blake3.workspace = true +chrono.workspace = true rusqlite.workspace = true +serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +utoipa.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs index 9863904c39..a6934e5168 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs @@ -62,10 +62,6 @@ pub(crate) fn key_text(value: Option<&RecallValue>, column_type: ColumnType) -> Some(RecallValue::Text(text)) if column_type == ColumnType::Timestamp => { parse_timestamp(text).map_or_else(|| text.clone(), format_timestamp) } - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), + _ => value.map_or_else(String::new, RecallValue::key_text), } } diff --git a/crates/sqlbuild-recall/src/clickhouse/constants.rs b/crates/sqlbuild-recall/src/clickhouse/constants.rs index d11b50dc61..2c8f2719df 100644 --- a/crates/sqlbuild-recall/src/clickhouse/constants.rs +++ b/crates/sqlbuild-recall/src/clickhouse/constants.rs @@ -61,8 +61,6 @@ pub(crate) const MAX_DATETIME64_PRECISION: u64 = 9; pub(crate) const MICROS_DIGITS: u64 = 6; /// One decimal digit of timestamp scale. pub(crate) const DECIMAL_SCALE: i64 = 10; -/// The column the store sets when a row is written. -pub(crate) const INGESTED_AT_COLUMN: &str = "ingested_at"; pub(crate) const VARINT_PAYLOAD_BITS: u32 = 7; pub(crate) const VARINT_CONTINUE: u8 = 0x80; pub(crate) const VARINT_MASK: u64 = 0x7f; diff --git a/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs index a91f7a0e35..7bd512fa2a 100644 --- a/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs +++ b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs @@ -1,9 +1,10 @@ //! `encode_rows`: one table's rows as a `RowBinary` insert block. use crate::clickhouse::_helpers::row_binary::{encoded, scaled}; -use crate::clickhouse::constants::{INGESTED_AT_COLUMN, MICROS_DIGITS}; +use crate::clickhouse::constants::MICROS_DIGITS; use crate::clickhouse::models::{ColumnEncoding, StoredColumn}; use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::constants::INGESTED_AT_COLUMN; /// The column names and `RowBinary` block of `rows`, all of one table, written with `ingested_at`. Values for columns the table lacks are left out; columns the rows lack take their defaults. pub fn encode_rows( diff --git a/crates/sqlbuild-recall/src/records/_helpers/mod.rs b/crates/sqlbuild-recall/src/records/_helpers/mod.rs index eaf86fcc66..e04c007fa0 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod events; pub(crate) mod ids; +pub(crate) mod record_row; pub(crate) mod rows; pub(crate) mod time; diff --git a/crates/sqlbuild-recall/src/records/_helpers/record_row.rs b/crates/sqlbuild-recall/src/records/_helpers/record_row.rs new file mode 100644 index 0000000000..3a99f89e08 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/record_row.rs @@ -0,0 +1,61 @@ +//! Decode transported record values against the physical schema, then compute their digest. + +use std::collections::BTreeMap; + +use crate::records::_helpers::rows::finish_row; +use crate::records::errors::RecordError; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{ColumnType, Table}; + +/// Envelope and digest columns belong to the receiver, never the transported values. +pub(crate) fn record_row( + envelope: &RowEnvelope, + table: Table, + values: &BTreeMap, +) -> Result { + let columns = table_spec(table).columns; + let mut decoded = Vec::with_capacity(values.len()); + for (name, value) in values { + let column = columns + .iter() + .find(|column| column.name == name) + .ok_or_else(|| RecordError::new(format!("unknown Recall record column {name}")))?; + let valid = match (value, column.column_type) { + (RecallValue::Null, _) => column.nullable, + (RecallValue::Integer(0 | 1), ColumnType::Boolean) => true, + (RecallValue::Integer(_), ColumnType::Integer) => true, + (RecallValue::Real(value), ColumnType::Real) => value.is_finite(), + (RecallValue::Blob(_), ColumnType::Blob) => true, + (RecallValue::Text(_), ColumnType::Text) => true, + (RecallValue::Text(value), ColumnType::Json) => { + serde_json::from_str::(value).map_err(|error| { + RecordError::new(format!("invalid Recall JSON column {name}: {error}")) + })?; + true + } + (RecallValue::Text(value), ColumnType::Timestamp) => { + chrono::DateTime::parse_from_rfc3339(value).map_err(|error| { + RecordError::new(format!("invalid Recall timestamp column {name}: {error}")) + })?; + true + } + _ => false, + }; + if !valid { + return Err(RecordError::new(format!( + "invalid Recall record column {name}" + ))); + } + decoded.push((column.name, value.clone())); + } + for column in columns.iter().filter(|column| !column.nullable) { + if !values.contains_key(column.name) { + return Err(RecordError::new(format!( + "missing Recall record column {}", + column.name + ))); + } + } + Ok(finish_row(table, envelope, decoded)) +} diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index e9da820c87..eaab7e2c60 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -3,7 +3,8 @@ use crate::schema::models::Table; /// One stored value. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] pub enum RecallValue { Null, Integer(i64), @@ -12,6 +13,20 @@ pub enum RecallValue { Blob(Vec), } +impl RecallValue { + /// Human-readable conflict-key text, before dialect-specific timestamp normalization. + #[must_use] + pub fn key_text(&self) -> String { + match self { + Self::Text(text) => text.clone(), + Self::Integer(number) => number.to_string(), + Self::Real(number) => number.to_string(), + Self::Blob(_) => "".to_owned(), + Self::Null => String::new(), + } + } +} + /// One row bound for a Recall table: named values, without `ingested_at`, which the store sets. #[derive(Clone, Debug, PartialEq)] pub struct RecallRow { @@ -20,6 +35,15 @@ pub struct RecallRow { } impl RecallRow { + /// Validate schema-typed values and rebuild the receiver-owned envelope and content digest. + pub fn from_values( + envelope: &RowEnvelope, + table: Table, + values: &std::collections::BTreeMap, + ) -> Result { + crate::records::_helpers::record_row::record_row(envelope, table, values) + } + /// The value of `column`, if the row has it. #[must_use] pub fn value(&self, column: &str) -> Option<&RecallValue> { @@ -48,7 +72,10 @@ pub struct RowEnvelope { } /// A process output stream. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] pub enum OutputStream { Stdout, Stderr, @@ -75,7 +102,8 @@ impl OutputStream { } /// One exact-byte output chunk. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] pub struct OutputChunk { /// Per invocation, from 0, shared by both streams. pub sequence: u64, @@ -91,7 +119,18 @@ pub struct OutputChunk { } /// Output of one stream that was not recorded because it exceeded `max_output_bytes`. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive( + Clone, + Copy, + Debug, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, +)] +#[serde(deny_unknown_fields)] pub struct OmittedOutput { pub bytes: u64, /// Sequences this gap accounts for, its own included. diff --git a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs index 275383fa1c..6282151cb0 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod clickhouse_views; +pub(crate) mod relational_tables; +pub(crate) mod relational_views; pub(crate) mod rendering; -pub(crate) mod sqlite_views; pub(crate) mod version_row; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs new file mode 100644 index 0000000000..5430d20356 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_tables.rs @@ -0,0 +1,60 @@ +//! One logical schema and index list for SQLite and PostgreSQL. + +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + +use crate::schema::_helpers::rendering::{all_columns, sqlite_type}; +use crate::schema::constants::{ARTIFACTS_TABLE, EVENTS_TABLE}; +use crate::schema::models::{ColumnType, TableSpec}; + +pub(crate) fn relational_table(spec: &TableSpec, postgres: bool, control: bool) -> Vec { + let mut definitions: Vec = all_columns(spec) + .iter() + .map(|column| { + let not_null = if column.nullable { "" } else { " NOT NULL" }; + let kind = if postgres { + column.column_type.postgres_type() + } else if control && column.column_type == ColumnType::Json { + "JSON" + } else { + sqlite_type(column.column_type) + }; + format!(" {} {kind}{not_null}", sqlite_identifier(column.name)) + }) + .collect(); + if control && spec.name == EVENTS_TABLE.name { + definitions.push(if postgres { + " storage_id BIGINT GENERATED ALWAYS AS IDENTITY".into() + } else { + " storage_id INTEGER PRIMARY KEY AUTOINCREMENT".into() + }); + } + let mut key: Vec<&str> = spec.key.to_vec(); + let partition = if postgres && spec.name != ARTIFACTS_TABLE.name { + if !key.contains(&spec.expiry_column) { + key.push(spec.expiry_column); + } + format!( + " PARTITION BY RANGE ({})", + sqlite_identifier(spec.expiry_column) + ) + } else { + String::new() + }; + let key: Vec = key.into_iter().map(sqlite_identifier).collect(); + definitions.push(format!(" UNIQUE ({})", key.join(", "))); + let mut statements = vec![format!( + "CREATE TABLE IF NOT EXISTS {} (\n{}\n){partition}", + sqlite_identifier(spec.name), + definitions.join(",\n") + )]; + for (suffix, columns) in spec.sqlite_indexes { + let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); + statements.push(format!( + "CREATE INDEX IF NOT EXISTS {} ON {} ({})", + sqlite_identifier(&format!("{}_{suffix}", spec.name)), + sqlite_identifier(spec.name), + columns.join(", ") + )); + } + statements +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs similarity index 60% rename from crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs rename to crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs index 8d4e1c2f81..321bdb28a8 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/relational_views.rs @@ -1,11 +1,15 @@ -//! The bodies of the supported views in SQLite. +//! Shared relational views emit dialect differences at each expression's definition. +use crate::schema::_helpers::rendering::{json_field, relational_cast}; use crate::schema::constants::OUTPUT_TAIL_RECORDS; -/// The current time as stored timestamps spell it. -const NOW: &str = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; +const EVENTS: &str = "SELECT * FROM recall_events"; +const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; +const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; -const STATEMENTS: &str = "WITH facts AS ( +fn statements(postgres: bool) -> String { + let available = relational_cast(postgres, ("artifact.digest IS NOT NULL", "INTEGER")); + format!("WITH facts AS ( SELECT * FROM recall_events WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' @@ -41,7 +45,7 @@ terminals AS ( ), captures AS ( SELECT firsts.statement_id, ( - SELECT MAX(artifact.digest IS NOT NULL) + SELECT MAX({available}) FROM recall_artifact_refs AS ref LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest WHERE ref.invocation_id = firsts.invocation_id @@ -102,9 +106,13 @@ LEFT JOIN captures ON captures.statement_id = firsts.statement_id LEFT JOIN resources ON resources.invocation_id = firsts.invocation_id AND resources.resource_id = firsts.resource_id -WHERE firsts.position = 1"; +WHERE firsts.position = 1") +} -const AUDIT_RESULTS: &str = "SELECT +fn audit_results(postgres: bool) -> String { + let field = |path, cast| json_field(postgres, ("payload", path), cast); + format!( + "SELECT invocation_id, invocation_started_at, event_id, @@ -112,40 +120,63 @@ const AUDIT_RESULTS: &str = "SELECT run_id, resource_id, resource_attempt_id, - json_extract(payload, '$.audit_name') AS audit_name, - json_extract(payload, '$.audit_definition_name') AS audit_definition_name, - CASE WHEN json_extract(payload, '$.target_name') IS NULL THEN NULL ELSE - COALESCE(json_extract(payload, '$.target_database') || '.', '') - || COALESCE(json_extract(payload, '$.target_schema') || '.', '') - || json_extract(payload, '$.target_name') + {audit_name} AS audit_name, + {audit_definition_name} AS audit_definition_name, + CASE WHEN {target_name} IS NULL THEN NULL ELSE + COALESCE({target_database} || '.', '') + || COALESCE({target_schema} || '.', '') + || {target_name} END AS target_relation, - json_extract(payload, '$.attached_target_kind') AS attached_target_kind, - json_extract(payload, '$.attached_target_name') AS attached_target_name, - json_extract(payload, '$.attached_column_name') AS \"column\", + {attached_target_kind} AS attached_target_kind, + {attached_target_name} AS attached_target_name, + {attached_column_name} AS \"column\", outcome, - json_extract(payload, '$.severity') AS severity, - json_extract(payload, '$.violation_count') AS violation_count, - json_extract(payload, '$.measured_value') AS measured_value, - json_extract(payload, '$.evaluation_mode') AS evaluation_mode, - json_extract(payload, '$.execution_error') AS execution_error, + {severity} AS severity, + {violation_count} AS violation_count, + {measured_value} AS measured_value, + {evaluation_mode} AS evaluation_mode, + {execution_error} AS execution_error, sql_digest, - json_extract(payload, '$.evidence_count') AS evidence_count, - json_extract(payload, '$.evidence_truncated') AS evidence_truncated, - json_extract(payload, '$.sample_digest') AS sample_digest + {evidence_count} AS evidence_count, + {evidence_truncated} AS evidence_truncated, + {sample_digest} AS sample_digest FROM recall_events -WHERE event_type = 'audit_completed'"; +WHERE event_type = 'audit_completed'", + audit_name = field("audit_name", ""), + audit_definition_name = field("audit_definition_name", ""), + target_name = field("target_name", ""), + target_database = field("target_database", ""), + target_schema = field("target_schema", ""), + attached_target_kind = field("attached_target_kind", ""), + attached_target_name = field("attached_target_name", ""), + attached_column_name = field("attached_column_name", ""), + severity = field("severity", ""), + violation_count = field("violation_count", "::BIGINT"), + measured_value = field("measured_value", "::DOUBLE PRECISION"), + evaluation_mode = field("evaluation_mode", ""), + execution_error = field("execution_error", ""), + evidence_count = field("evidence_count", "::BIGINT"), + evidence_truncated = field("evidence_truncated", "::BOOLEAN::INTEGER"), + sample_digest = field("sample_digest", "") + ) +} -const OUTPUT_INVOCATIONS: &str = "WITH received AS ( +fn output_invocations(postgres: bool) -> String { + let sum = |expression| relational_cast(postgres, (&format!("SUM({expression})"), "BIGINT")); + let maximum = if postgres { "GREATEST" } else { "MAX" }; + let filter = relational_cast(postgres, ("context.recall_filter_active", "INTEGER")); + let flush = relational_cast(postgres, ("delivery.flush_complete", "INTEGER")); + format!("WITH received AS ( SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at, COUNT(*) AS chunks, - SUM(byte_count) AS bytes, - SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS stdout_bytes, - SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS stderr_bytes, + {bytes} AS bytes, + {stdout_bytes} AS stdout_bytes, + {stderr_bytes} AS stderr_bytes, MAX(sequence) AS max_sequence, - SUM(dropped_before) AS queue_drops, - SUM(COALESCE(omitted_sequences, 1)) AS spans, - SUM(COALESCE(omitted_bytes, 0)) AS truncated_bytes + {drops} AS queue_drops, + {spans} AS spans, + {truncated} AS truncated_bytes FROM recall_output GROUP BY invocation_id ), @@ -166,15 +197,15 @@ counted AS ( COALESCE(received.stderr_bytes, 0) AS stderr_bytes, COALESCE(received.truncated_bytes, 0) AS truncated_bytes, received.max_sequence, - MAX( + {maximum}( COALESCE(received.max_sequence + 1 - received.spans, 0), COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) ) AS missing_sequences, - MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + {maximum}(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, COALESCE(delivery.filtered, 0) AS filtered, COALESCE(delivery.failed, 0) AS failed, - COALESCE(context.recall_filter_active, 0) AS filter_active, - COALESCE(delivery.flush_complete, 0) AS flush_complete, + COALESCE({filter}, 0) AS filter_active, + COALESCE({flush}, 0) AS flush_complete, delivery.invocation_id IS NOT NULL AS delivery_reported, COALESCE(delivery.delivered, 0) AS delivered FROM ids @@ -210,26 +241,21 @@ SELECT AND chunks >= delivered THEN 1 ELSE 0 END AS complete -FROM counted"; - -const OUTPUT_TRANSCRIPTS: &str = "SELECT - invocation_id, - MIN(invocation_started_at) AS invocation_started_at, - CAST(COALESCE( - group_concat(CASE WHEN stream = 'stdout' THEN data END, x'' ORDER BY sequence), x'' - ) AS BLOB) AS stdout, - CAST(COALESCE( - group_concat(CASE WHEN stream = 'stderr' THEN data END, x'' ORDER BY sequence), x'' - ) AS BLOB) AS stderr, - CAST(COALESCE(group_concat(data, x'' ORDER BY sequence), x'') AS BLOB) AS combined -FROM recall_output -GROUP BY invocation_id"; - -const EVENTS: &str = "SELECT * FROM recall_events"; - -const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; +FROM counted", + bytes = sum("byte_count"), stdout_bytes = sum("CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END"), + stderr_bytes = sum("CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END"), drops = sum("dropped_before"), + spans = sum("COALESCE(omitted_sequences, 1)"), truncated = sum("COALESCE(omitted_bytes, 0)")) +} -const ARTIFACTS: &str = "SELECT +fn artifacts(postgres: bool) -> String { + let available = relational_cast(postgres, ("artifact.digest IS NOT NULL", "INTEGER")); + let text = if postgres { + "artifact.content" + } else { + "CAST(artifact.content AS TEXT)" + }; + format!( + "SELECT refs.invocation_id, refs.invocation_started_at, refs.digest, @@ -240,17 +266,61 @@ const ARTIFACTS: &str = "SELECT artifact.kind, artifact.media_type, artifact.size, - artifact.digest IS NOT NULL AS available, + {available} AS available, CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' - THEN CAST(artifact.content AS TEXT) END AS text, + THEN {text} END AS text, artifact.content FROM recall_artifact_refs AS refs -LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest" + ) +} -const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; +fn output_transcripts(postgres: bool) -> String { + let columns: Vec = [ + ("CASE WHEN stream = 'stdout' THEN data END", "stdout"), + ("CASE WHEN stream = 'stderr' THEN data END", "stderr"), + ("data", "combined"), + ] + .into_iter() + .map(|(expression, alias)| { + let value = if postgres { + format!("COALESCE(string_agg({expression}, ''::BYTEA ORDER BY sequence), ''::BYTEA)") + } else { + format!( + "CAST(COALESCE(group_concat({expression}, x'' ORDER BY sequence), x'') AS BLOB)" + ) + }; + format!(" {value} AS {alias}") + }) + .collect(); + format!( + "SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at,\n{}\nFROM recall_output GROUP BY invocation_id", + columns.join(",\n") + ) +} -fn invocations() -> String { +fn invocations(postgres: bool) -> String { + let now = if postgres { + "CURRENT_TIMESTAMP" + } else { + "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')" + }; + let selection_count = if postgres { + "jsonb_array_length(context.selection)" + } else { + "json_array_length(context.selection)" + }; + let valid_context = if postgres { + "context.external_context IS NOT NULL" + } else { + "json_valid(context.external_context)" + }; + let elapsed = if postgres { + "CAST(ROUND(CAST(EXTRACT(EPOCH FROM (last_heartbeat_at - started_at)) * 1000 AS NUMERIC), 3) AS DOUBLE PRECISION)" + } else { + "ROUND((unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3)" + }; format!( "WITH ids AS ( SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at @@ -259,7 +329,7 @@ fn invocations() -> String { UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') - ) + ) AS source_rows GROUP BY invocation_id ), started AS ( @@ -270,9 +340,9 @@ started AS ( ), terminals AS ( SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, - json_extract(payload, '$.error_message') AS error_message, - json_extract(payload, '$.signal_number') AS signal_number, - json_extract(payload, '$.cost_status') AS cost_status, + {error_message} AS error_message, + {signal_number} AS signal_number, + {cost_status} AS cost_status, ROW_NUMBER() OVER ( PARTITION BY invocation_id ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC @@ -297,7 +367,7 @@ projected AS ( AND terminals.signal_number IS NOT NULL THEN 'cancelled' WHEN terminals.event_type = 'invocation_failed' THEN 'failed' WHEN beats.stale_at IS NULL THEN 'unknown' - WHEN beats.stale_at >= {NOW} THEN 'running' + WHEN beats.stale_at >= {now} THEN 'running' ELSE 'abandoned' END AS status, terminals.event_type AS terminal_event, @@ -317,7 +387,7 @@ projected AS ( context.adapter, context.project_name, context.selection, - COALESCE(context.selection_count, json_array_length(context.selection)) + COALESCE(context.selection_count, {selection_count}) AS selection_count, context.selection_truncated, context.selector_files, @@ -328,17 +398,17 @@ projected AS ( context.host_token, context.pid, context.external_context, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.name') END + CASE WHEN {valid_context} + THEN {integration} END AS integration, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.job_name') END + CASE WHEN {valid_context} + THEN {integration_job} END AS integration_job, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.run_id') END + CASE WHEN {valid_context} + THEN {integration_run_id} END AS integration_run_id, - CASE WHEN json_valid(context.external_context) - THEN json_extract(context.external_context, '$.integration.retry_number') END + CASE WHEN {valid_context} + THEN {integration_retry} END AS integration_retry, context.recall_filter_active, beats.last_heartbeat_at @@ -360,9 +430,7 @@ SELECT END AS ended_at, CASE WHEN terminal_event IS NOT NULL THEN terminal_duration_ms - WHEN status = 'abandoned' THEN ROUND( - (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3 - ) + WHEN status = 'abandoned' THEN {elapsed} END AS duration_ms, CASE WHEN terminal_event IS NOT NULL THEN 0 @@ -398,34 +466,62 @@ SELECT integration_retry, recall_filter_active, last_heartbeat_at -FROM projected" +FROM projected", + error_message = json_field(postgres, ("payload", "error_message"), ""), + signal_number = json_field(postgres, ("payload", "signal_number"), "::BIGINT"), + cost_status = json_field(postgres, ("payload", "cost_status"), ""), + integration = json_field( + postgres, + ("context.external_context", "integration.name"), + "" + ), + integration_job = json_field( + postgres, + ("context.external_context", "integration.job_name"), + "" + ), + integration_run_id = json_field( + postgres, + ("context.external_context", "integration.run_id"), + "" + ), + integration_retry = json_field( + postgres, + ("context.external_context", "integration.retry_number"), + "::BIGINT" + ) ) } -fn output_tail() -> String { +fn output_tail(postgres: bool) -> String { + let text = if postgres { + "data" + } else { + "CAST(data AS TEXT)" + }; format!( "SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, - CAST(data AS TEXT) AS text + {text} AS text FROM ( SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position FROM recall_output -) +) AS source_rows WHERE position <= {OUTPUT_TAIL_RECORDS}" ) } -/// Each supported view's name and SQLite body, in creation order. -pub(crate) fn sqlite_view_bodies() -> Vec<(&'static str, String)> { +/// Each supported view's name and relational body, in creation order. +pub(crate) fn relational_view_bodies(postgres: bool) -> Vec<(&'static str, String)> { vec![ - ("invocations", invocations()), - ("statements", STATEMENTS.to_owned()), - ("audit_results", AUDIT_RESULTS.to_owned()), - ("output_invocations", OUTPUT_INVOCATIONS.to_owned()), - ("output_tail", output_tail()), - ("output_transcripts", OUTPUT_TRANSCRIPTS.to_owned()), + ("invocations", invocations(postgres)), + ("statements", statements(postgres)), + ("audit_results", audit_results(postgres)), + ("output_invocations", output_invocations(postgres)), + ("output_tail", output_tail(postgres)), + ("output_transcripts", output_transcripts(postgres)), ("events", EVENTS.to_owned()), ("diagnostics", DIAGNOSTICS.to_owned()), - ("artifacts", ARTIFACTS.to_owned()), + ("artifacts", artifacts(postgres)), ("conflicts", CONFLICTS.to_owned()), ] } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs index 658aef267d..f7355f0442 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs @@ -36,11 +36,6 @@ pub(crate) fn clickhouse_type(column: &ColumnSpec) -> String { } } -/// `"name"` for SQLite. -pub(crate) fn sqlite_identifier(name: &str) -> String { - format!("\"{}\"", name.replace('"', "\"\"")) -} - /// `` `name` `` for ClickHouse. pub(crate) fn clickhouse_identifier(name: &str) -> String { format!("`{}`", name.replace('\\', "\\\\").replace('`', "\\`")) @@ -73,6 +68,22 @@ pub(crate) fn clickhouse_ttl(spec: &TableSpec, options: &DdlOptions) -> String { format!("{} + INTERVAL {days} DAY", spec.expiry_column) } +pub(crate) fn json_field(postgres: bool, (column, path): (&str, &str), cast: &str) -> String { + if postgres { + format!("({column} #>> '{{{}}}'){cast}", path.replace('.', ",")) + } else { + format!("json_extract({column}, '$.{path}')") + } +} + +pub(crate) fn relational_cast(postgres: bool, (expression, kind): (&str, &str)) -> String { + if postgres { + format!("CAST({expression} AS {kind})") + } else { + expression.into() + } +} + /// The engine clause, replicated with one shard and every replica on a cluster. pub(crate) fn clickhouse_engine(spec: &TableSpec, options: &DdlOptions) -> String { let version: &str = spec.clickhouse_version.unwrap_or_default(); diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 48e5a69a36..05c9c15d98 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -17,13 +17,16 @@ pub const DDL_SCRIPT_INVOCATION: &str = "ddl-script"; pub const META_MAINTENANCE_COMPONENT: &str = "maintenance"; pub const META_MAINTENANCE_CLAIM_KEY: &str = "claim"; +/// Assigned by storage rather than accepted from a producer's row values. +pub const INGESTED_AT_COLUMN: &str = "ingested_at"; + /// Columns every Recall row carries, after the table's own columns. pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ ColumnSpec::required("invocation_id", ColumnType::Text), ColumnSpec::required("invocation_started_at", ColumnType::Timestamp), ColumnSpec::required("schema_version", ColumnType::Integer), ColumnSpec::required("producer_version", ColumnType::Text), - ColumnSpec::required("ingested_at", ColumnType::Timestamp), + ColumnSpec::required(INGESTED_AT_COLUMN, ColumnType::Timestamp), ColumnSpec::required("content_hash", ColumnType::Text), ]; /// The artifact columns its content digest covers: what it is, not who saw it when. diff --git a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs index cea0815488..dc69488373 100644 --- a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs +++ b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs @@ -1,8 +1,9 @@ //! `add_column_statement`: the additive upgrade of one table by one column. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::rendering::{ - clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_identifier, - sqlite_type, + clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_type, }; use crate::schema::models::{ColumnSpec, DdlOptions, Dialect, TableSpec}; diff --git a/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs new file mode 100644 index 0000000000..db06b995c6 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/control_table_statements.rs @@ -0,0 +1,14 @@ +//! Control stores preserve Recall's logical columns and add a committed-order event cursor. + +use crate::schema::_helpers::relational_tables::relational_table; +use crate::schema::constants::ALL_TABLES; +use crate::schema::main::table_spec::table_spec; + +/// PostgreSQL partitions invocation facts; globally deduplicated artifacts remain unpartitioned. +#[must_use] +pub fn control_table_statements(postgres: bool) -> Vec { + ALL_TABLES + .iter() + .flat_map(|table| relational_table(table_spec(*table), postgres, true)) + .collect() +} diff --git a/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs new file mode 100644 index 0000000000..1645d1ab19 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/control_view_statements.rs @@ -0,0 +1,35 @@ +//! Control stores expose Recall's supported views on both engines. + +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + +use crate::schema::_helpers::relational_views::relational_view_bodies; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{DdlOptions, Dialect}; + +/// PostgreSQL validates each view at migration time; SQLite retains its existing view contract. +#[must_use] +pub fn control_view_statements(postgres: bool) -> Vec { + if postgres { + relational_view_bodies(true) + .into_iter() + .map(|(name, body)| { + format!( + "CREATE OR REPLACE VIEW {} AS\n{body}", + sqlite_identifier(name) + ) + }) + .collect() + } else { + view_statements( + Dialect::Sqlite, + &DdlOptions { + database: None, + retention_days: 30, + cluster: None, + }, + ) + .into_iter() + .map(|(_, statement)| statement) + .collect() + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs index e2251bd838..879cacea23 100644 --- a/crates/sqlbuild-recall/src/schema/main/mod.rs +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -1,8 +1,11 @@ pub mod add_column_statement; pub mod clickhouse_relation; +pub mod control_table_statements; +pub mod control_view_statements; pub mod create_database_statement; pub mod ddl_script; pub mod ddl_statements; +pub mod postgres_partition_statement; pub mod render_script; pub mod retention_fix_statements; pub mod schema_age; diff --git a/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs new file mode 100644 index 0000000000..4d24b5fbdb --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/postgres_partition_statement.rs @@ -0,0 +1,24 @@ +//! Monthly partition bounds are calculated from a typed date, never interpolated user SQL. + +use chrono::{Datelike, Months, NaiveDate}; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::Table; + +/// A replay uses the same timestamp partition and therefore the same deterministic unique key. +#[must_use] +pub fn postgres_partition_statement(table: Table, date: NaiveDate) -> Option { + if table == Table::Artifacts { + return None; + } + let first = date.with_day(1)?; + let next = first.checked_add_months(Months::new(1))?; + let spec = table_spec(table); + let partition = format!("{}_{}", spec.name, first.format("%Y%m")); + Some(format!( + "CREATE TABLE IF NOT EXISTS {} PARTITION OF {} FOR VALUES FROM ('{first}T00:00:00Z') TO ('{next}T00:00:00Z')", + sqlite_identifier(&partition), + sqlite_identifier(spec.name) + )) +} diff --git a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs index 1a44c7b2e4..44b5d5ee02 100644 --- a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs +++ b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs @@ -1,6 +1,7 @@ //! `sqlite_retention_sql`: the deletes SQLite maintenance runs, all rows of an invocation together. -use crate::schema::_helpers::rendering::sqlite_identifier; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::constants::{ INVOCATION_TABLES, META_MAINTENANCE_COMPONENT, RETENTION_CANDIDATE_TABLES, }; diff --git a/crates/sqlbuild-recall/src/schema/main/table_statements.rs b/crates/sqlbuild-recall/src/schema/main/table_statements.rs index 621f8e1f12..7829997b9b 100644 --- a/crates/sqlbuild-recall/src/schema/main/table_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/table_statements.rs @@ -1,48 +1,14 @@ //! `table_statements`: idempotent `CREATE TABLE` and index statements of every Recall table. +use crate::schema::_helpers::relational_tables::relational_table; use crate::schema::_helpers::rendering::{ all_columns, clickhouse_engine, clickhouse_identifier, clickhouse_table, clickhouse_ttl, - clickhouse_type, on_cluster, sqlite_identifier, sqlite_type, + clickhouse_type, on_cluster, }; use crate::schema::constants::ALL_TABLES; use crate::schema::main::table_spec::table_spec; use crate::schema::models::{DdlOptions, Dialect, TableSpec}; -fn sqlite_table(spec: &TableSpec) -> Vec { - let mut definitions: Vec = all_columns(spec) - .iter() - .map(|column| { - let not_null: &str = if column.nullable { "" } else { " NOT NULL" }; - format!( - " {} {}{not_null}", - sqlite_identifier(column.name), - sqlite_type(column.column_type) - ) - }) - .collect(); - let key: Vec = spec - .key - .iter() - .map(|name| sqlite_identifier(name)) - .collect(); - definitions.push(format!(" UNIQUE ({})", key.join(", "))); - let mut statements: Vec = vec![format!( - "CREATE TABLE IF NOT EXISTS {} (\n{}\n)", - sqlite_identifier(spec.name), - definitions.join(",\n") - )]; - for (suffix, columns) in spec.sqlite_indexes { - let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); - statements.push(format!( - "CREATE INDEX IF NOT EXISTS {} ON {} ({})", - sqlite_identifier(&format!("{}_{suffix}", spec.name)), - sqlite_identifier(spec.name), - columns.join(", ") - )); - } - statements -} - fn clickhouse_table_statement(spec: &TableSpec, options: &DdlOptions) -> String { let columns: Vec = all_columns(spec) .iter() @@ -76,7 +42,7 @@ pub fn table_statements(dialect: Dialect, options: &DdlOptions) -> Vec { .flat_map(|table| { let spec: &TableSpec = table_spec(*table); match dialect { - Dialect::Sqlite => sqlite_table(spec), + Dialect::Sqlite => relational_table(spec, false, false), Dialect::ClickHouse => vec![clickhouse_table_statement(spec, options)], } }) diff --git a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs index 0b8c7430e2..913bf845df 100644 --- a/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/upgrade_statements.rs @@ -1,6 +1,7 @@ //! `upgrade_statements`: what an administrator runs to bring an observed store up to this build's schema. -use crate::schema::_helpers::rendering::sqlite_identifier; +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::version_row::version_row_statement; use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION}; use crate::schema::main::add_column_statement::add_column_statement; diff --git a/crates/sqlbuild-recall/src/schema/main/view_statements.rs b/crates/sqlbuild-recall/src/schema/main/view_statements.rs index ce34504a3c..0351938b81 100644 --- a/crates/sqlbuild-recall/src/schema/main/view_statements.rs +++ b/crates/sqlbuild-recall/src/schema/main/view_statements.rs @@ -1,15 +1,17 @@ //! `view_statements`: the statements that create the supported views. +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + use crate::schema::_helpers::clickhouse_views::clickhouse_view_bodies; -use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster, sqlite_identifier}; -use crate::schema::_helpers::sqlite_views::sqlite_view_bodies; +use crate::schema::_helpers::relational_views::relational_view_bodies; +use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster}; use crate::schema::models::{DdlOptions, Dialect}; /// Each view's name and creation statement. SQLite views are created only when missing; the store drops them first when the schema version rises. ClickHouse views are replaced. #[must_use] pub fn view_statements(dialect: Dialect, options: &DdlOptions) -> Vec<(&'static str, String)> { match dialect { - Dialect::Sqlite => sqlite_view_bodies() + Dialect::Sqlite => relational_view_bodies(false) .into_iter() .map(|(name, body)| { ( diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs index 022e4f0b84..e622f273fe 100644 --- a/crates/sqlbuild-recall/src/schema/models.rs +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -22,6 +22,22 @@ pub enum ColumnType { Boolean, } +impl ColumnType { + /// The PostgreSQL type used by both DDL and typed query placeholders. + #[must_use] + pub const fn postgres_type(self) -> &'static str { + match self { + Self::Text => "TEXT", + Self::Timestamp => "TIMESTAMPTZ", + Self::Json => "JSONB", + Self::Integer => "BIGINT", + Self::Boolean => "BOOLEAN", + Self::Real => "DOUBLE PRECISION", + Self::Blob => "BYTEA", + } + } +} + /// One column: its name, type and whether it may be null. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ColumnSpec { @@ -75,7 +91,20 @@ pub struct TableSpec { } /// The tables Recall writes. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, + utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] pub enum Table { Invocations, Output, diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs index 3344c9a183..fab996484f 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod column_types; pub(crate) mod identity; pub(crate) mod quarantine; +pub(crate) mod relational; pub(crate) mod rows; pub(crate) mod schema; pub(crate) mod sqlite; diff --git a/crates/sqlbuild-recall/src/store/_helpers/relational.rs b/crates/sqlbuild-recall/src/store/_helpers/relational.rs new file mode 100644 index 0000000000..7122ec88b9 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/relational.rs @@ -0,0 +1,11 @@ +//! Typed placeholders shared by native SQLite and control-plane transactions. + +use crate::schema::models::ColumnType; + +pub(crate) fn placeholder(index: usize, kind: ColumnType, postgres: bool) -> String { + if postgres { + format!("CAST(${index} AS {})", kind.postgres_type()) + } else { + format!("${index}") + } +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs index 5fed0c3900..aa5b4d0b42 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -12,20 +12,14 @@ use crate::schema::models::Table; use crate::store::_helpers::schema::stored_columns; use crate::store::_helpers::sqlite::{SqlValue, checked}; use crate::store::errors::StoreError; +use crate::store::main::conflict_evidence::conflict_evidence; +use crate::store::main::relational_insert::relational_insert; +use crate::store::main::relational_lookup::relational_lookup; use crate::store::models::{ConflictEvidence, InsertOutcome}; -fn key_text(value: Option<&RecallValue>) -> String { - match value { - Some(RecallValue::Text(text)) => text.clone(), - Some(RecallValue::Integer(number)) => number.to_string(), - Some(RecallValue::Real(number)) => number.to_string(), - Some(RecallValue::Blob(_)) => "".to_owned(), - Some(RecallValue::Null) | None => String::new(), - } -} - fn text_of(row: &RecallRow, column: &str) -> String { - key_text(row.value(column)) + row.value(column) + .map_or_else(String::new, RecallValue::key_text) } /// Insert one row's values for the columns its table has; `Some(evidence)` when its key holds different content. @@ -34,28 +28,17 @@ fn write_row( (row, columns): (&RecallRow, &HashSet), ingested_at: &str, ) -> Result, StoreError> { - let spec = table_spec(row.table); let ingested: RecallValue = RecallValue::Text(ingested_at.to_owned()); - let stored: Vec<&(&str, RecallValue)> = row + let insert = relational_insert((row, columns), &ingested, false); + let parameters: Vec> = insert .values .iter() - .filter(|(name, _)| columns.contains(*name)) + .map(|(value, _)| SqlValue(value)) .collect(); - let mut names: Vec<&str> = stored.iter().map(|(name, _)| *name).collect(); - names.push("ingested_at"); - let placeholders: Vec = (1..=names.len()).map(|index| format!("?{index}")).collect(); - let sql = format!( - "INSERT INTO \"{}\" ({}) VALUES ({}) ON CONFLICT DO NOTHING", - spec.name, - names.join(", "), - placeholders.join(", ") - ); - let parameters: Vec> = stored - .iter() - .map(|(_, value)| SqlValue(value)) - .chain(std::iter::once(SqlValue(&ingested))) - .collect(); - let mut statement = checked(connection.prepare_cached(&sql), "prepare Recall insert")?; + let mut statement = checked( + connection.prepare_cached(&insert.sql), + "prepare Recall insert", + )?; let changed: usize = checked( statement.execute(rusqlite::params_from_iter(parameters.iter())), "write Recall rows", @@ -63,24 +46,16 @@ fn write_row( if changed > 0 { return Ok(None); } - let condition: Vec = spec - .key - .iter() - .enumerate() - .map(|(index, name)| format!("{name} = ?{}", index + 1)) - .collect(); - let lookup = format!( - "SELECT content_hash FROM \"{}\" WHERE {}", - spec.name, - condition.join(" AND ") - ); - let null = RecallValue::Null; - let key_values: Vec> = spec - .key + let lookup = relational_lookup(row, false, false); + let key_values: Vec> = lookup + .values .iter() - .map(|name| SqlValue(row.value(name).unwrap_or(&null))) + .map(|(value, _)| SqlValue(value)) .collect(); - let mut statement = checked(connection.prepare_cached(&lookup), "prepare Recall lookup")?; + let mut statement = checked( + connection.prepare_cached(&lookup.sql), + "prepare Recall lookup", + )?; let existing: Option = checked( statement .query_row(rusqlite::params_from_iter(key_values.iter()), |found| { @@ -89,21 +64,7 @@ fn write_row( .optional(), "read stored Recall row", )?; - let rejected: &str = row.content_hash().unwrap_or_default(); - match existing { - Some(existing) if existing != rejected => Ok(Some(ConflictEvidence { - table: row.table, - record_key: spec - .key - .iter() - .map(|name| key_text(row.value(name))) - .collect::>() - .join(":"), - existing_sha256: existing, - rejected_sha256: rejected.to_owned(), - })), - _ => Ok(None), - } + Ok(existing.and_then(|existing| conflict_evidence(row, &existing))) } /// Write `rows` on the current transaction; conflicts become `recall_conflicts` rows. diff --git a/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs new file mode 100644 index 0000000000..a4c04a92e6 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/conflict_evidence.rs @@ -0,0 +1,25 @@ +//! Duplicate rows preserve the first content and expose the rejected digest. + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::main::table_spec::table_spec; +use crate::store::models::ConflictEvidence; + +/// Evidence only when a stored row's content differs, independent of the storage driver. +#[must_use] +pub fn conflict_evidence(row: &RecallRow, existing: &str) -> Option { + let rejected = row.content_hash().unwrap_or_default(); + (existing != rejected).then(|| ConflictEvidence { + table: row.table, + record_key: table_spec(row.table) + .key + .iter() + .map(|name| { + row.value(name) + .map_or_else(String::new, RecallValue::key_text) + }) + .collect::>() + .join(":"), + existing_sha256: existing.to_owned(), + rejected_sha256: rejected.to_owned(), + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs index 69ccb1cf7e..78f9b89028 100644 --- a/crates/sqlbuild-recall/src/store/main/mod.rs +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -1,5 +1,6 @@ pub mod checkpoint_sqlite_store; pub mod chosen_invocation; +pub mod conflict_evidence; pub mod describe_store; pub mod ensure_sqlite_schema; pub mod insert_rows; @@ -10,5 +11,7 @@ pub mod open_sqlite_store; pub mod quarantine_store; pub mod query_store; pub mod read_output_transcript; +pub mod relational_insert; +pub mod relational_lookup; pub mod resolve_invocation; pub mod store_location_problem; diff --git a/crates/sqlbuild-recall/src/store/main/relational_insert.rs b/crates/sqlbuild-recall/src/store/main/relational_insert.rs new file mode 100644 index 0000000000..70aa341299 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/relational_insert.rs @@ -0,0 +1,50 @@ +//! One insert contract for standalone SQLite Recall and transactional control stores. + +use std::collections::HashSet; + +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::constants::INGESTED_AT_COLUMN; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::ColumnType; +use crate::store::_helpers::relational::placeholder; +use crate::store::models::RecallStatement; + +/// Ignore fields absent from an older additive schema; ingestion time is assigned by the store. +#[must_use] +pub fn relational_insert<'a>( + (row, columns): (&'a RecallRow, &HashSet), + ingested_at: &'a RecallValue, + postgres: bool, +) -> RecallStatement<'a> { + let metadata = table_columns(row.table); + let mut names: Vec = Vec::new(); + let mut values: Vec<(&RecallValue, ColumnType)> = Vec::new(); + for (name, value) in &row.values { + if let Some(column) = metadata.iter().find(|column| column.name == *name) + && columns.contains(*name) + && *name != INGESTED_AT_COLUMN + { + names.push(sqlite_identifier(name)); + values.push((value, column.column_type)); + } + } + names.push(INGESTED_AT_COLUMN.into()); + values.push((ingested_at, ColumnType::Timestamp)); + let placeholders: Vec = values + .iter() + .enumerate() + .map(|(index, (_, kind))| placeholder(index + 1, *kind, postgres)) + .collect(); + RecallStatement { + sql: format!( + "INSERT INTO {} ({}) VALUES ({}) ON CONFLICT DO NOTHING", + sqlite_identifier(table_spec(row.table).name), + names.join(", "), + placeholders.join(", ") + ), + values, + } +} diff --git a/crates/sqlbuild-recall/src/store/main/relational_lookup.rs b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs new file mode 100644 index 0000000000..8d6658a6d2 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/relational_lookup.rs @@ -0,0 +1,46 @@ +//! Duplicate detection uses the same stable keys as physical unique constraints. + +use sqlbuild_core::identifier_text::main::quote_identifier::quote_identifier as sqlite_identifier; + +use crate::records::models::{RecallRow, RecallValue}; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{ColumnType, Table}; +use crate::store::_helpers::relational::placeholder; +use crate::store::models::RecallStatement; + +/// Control event lookups return their durable cursor too; standalone stores have no cursor column. +#[must_use] +pub fn relational_lookup(row: &RecallRow, postgres: bool, control: bool) -> RecallStatement<'_> { + let spec = table_spec(row.table); + let mut keys = spec.key.to_vec(); + if postgres && row.table != Table::Artifacts && !keys.contains(&spec.expiry_column) { + keys.push(spec.expiry_column); + } + let metadata = table_columns(row.table); + let mut values: Vec<(&RecallValue, ColumnType)> = Vec::new(); + let mut conditions: Vec = Vec::new(); + for column in metadata.iter().filter(|column| keys.contains(&column.name)) { + let key = column.name; + let kind = column.column_type; + values.push((row.value(key).unwrap_or(&RecallValue::Null), kind)); + conditions.push(format!( + "{} = {}", + sqlite_identifier(key), + placeholder(values.len(), kind, postgres) + )); + } + let cursor = if control && row.table == Table::Events { + ", storage_id" + } else { + "" + }; + RecallStatement { + sql: format!( + "SELECT content_hash{cursor} FROM {} WHERE {}", + sqlite_identifier(spec.name), + conditions.join(" AND ") + ), + values, + } +} diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index a92d4ead30..fd53a8b5c7 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -4,7 +4,14 @@ use std::path::PathBuf; use std::time::{Duration, SystemTime}; use crate::records::models::{RecallValue, RowEnvelope}; -use crate::schema::models::Table; +use crate::schema::models::{ColumnType, Table}; + +/// A relational insert or lookup and its borrowed, typed parameters. +#[derive(Clone, Debug, PartialEq)] +pub struct RecallStatement<'a> { + pub sql: String, + pub values: Vec<(&'a RecallValue, ColumnType)>, +} /// One open store file. Only one thread uses a store. #[derive(Debug)] @@ -47,7 +54,7 @@ pub enum QuarantineOutcome { } /// A row whose key was already stored with different content; the first row is kept. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] pub struct ConflictEvidence { pub table: Table, pub record_key: String, @@ -56,7 +63,9 @@ pub struct ConflictEvidence { } /// What one insert batch did, per table. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive( + Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema, +)] pub struct InsertOutcome { /// Rows written or already present with equal content, by table. pub delivered: Vec<(Table, u64)>, diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 8083997c83..184ed9e83b 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -41,7 +41,7 @@ pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ pub const STRING_STATE_VALUE_TYPE: &str = "string"; /// State value type of a timestamp stored as ISO text. pub const TEXT_TIMESTAMP_STATE_VALUE_TYPE: &str = "text_timestamp"; -/// Source freshness state table columns in table order, with the state value type each declares. +/// Legacy latest-reader columns; the nullable event ID is intentionally not required by reads. pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ("source_name", STRING_STATE_VALUE_TYPE), ("target_database", STRING_STATE_VALUE_TYPE), @@ -54,5 +54,8 @@ pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ ("data_version_hash", STRING_STATE_VALUE_TYPE), ("observed_at", TEXT_TIMESTAMP_STATE_VALUE_TYPE), ]; +pub const SOURCE_FRESHNESS_EVENT_ID: &str = "event_id"; +pub const OBSERVED_AT_COLUMN: &str = "observed_at"; +pub const SQLITE_ADAPTER: &str = "sqlite"; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/event_id.rs b/crates/sqlbuild-state/src/freshness/main/event_id.rs new file mode 100644 index 0000000000..72bdeab470 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/event_id.rs @@ -0,0 +1,22 @@ +//! Observation identity excludes mutable payload and future storage columns. + +use sha2::{Digest, Sha256}; + +use crate::freshness::models::FreshnessEventIdentity; + +/// `observed_at` is UTC ISO text with six fractional digits and a `+00:00` offset. +pub fn event_id(row: &FreshnessEventIdentity<'_>) -> String { + let identity = serde_json::json!([ + "sqlbuild.source-freshness.event.v1", + row.source_name, + row.target_database, + row.target_schema, + row.target_name, + row.run_id, + row.observed_at, + row.strategy, + row.value_kind, + row.data_version_hash, + ]); + format!("{:x}", Sha256::digest(identity.to_string().as_bytes())) +} diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs index 9ec47ed663..2e5f2dbe8f 100644 --- a/crates/sqlbuild-state/src/freshness/main/mod.rs +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -1,6 +1,7 @@ pub mod age_status; pub mod configuration_error; pub mod data_version_hash; +pub mod event_id; pub mod normalized_data_version; pub mod read_latest_sql; pub mod records_equivalent; diff --git a/crates/sqlbuild-state/src/freshness/models.rs b/crates/sqlbuild-state/src/freshness/models.rs index dbe1341a71..9f7cbfb958 100644 --- a/crates/sqlbuild-state/src/freshness/models.rs +++ b/crates/sqlbuild-state/src/freshness/models.rs @@ -2,6 +2,21 @@ use sqlbuild_cursor::algebra::models::LocalDateTime; +/// Stable identity fields shared by warehouse writers and legacy history ingestion. +#[derive(Debug)] +pub struct FreshnessEventIdentity<'a> { + pub source_name: &'a str, + pub target_database: Option<&'a str>, + pub target_schema: Option<&'a str>, + pub target_name: Option<&'a str>, + pub run_id: &'a str, + pub strategy: &'a str, + pub value_kind: &'a str, + pub data_version_hash: &'a str, + /// UTC ISO text with six fractional digits and a `+00:00` offset. + pub observed_at: &'a str, +} + /// An observed data version of one value kind. #[derive(Clone, Debug, PartialEq, Eq)] pub enum DataVersion { diff --git a/crates/sqlbuild-state/src/state_tables/constants.rs b/crates/sqlbuild-state/src/state_tables/constants.rs new file mode 100644 index 0000000000..8bf6191df2 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/constants.rs @@ -0,0 +1,3 @@ +//! Adapter decisions shared by append-only state history reads. + +pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; diff --git a/crates/sqlbuild-state/src/state_tables/main/mod.rs b/crates/sqlbuild-state/src/state_tables/main/mod.rs index 712abe33fc..6b46ba4e98 100644 --- a/crates/sqlbuild-state/src/state_tables/main/mod.rs +++ b/crates/sqlbuild-state/src/state_tables/main/mod.rs @@ -1,2 +1,3 @@ pub mod create_table_sql; +pub mod read_history_sql; pub mod values_insert_sql; diff --git a/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs new file mode 100644 index 0000000000..2d871377ba --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs @@ -0,0 +1,119 @@ +//! Keyset pages preserve distinct observations without rereading or ranking all preceding rows. + +use crate::state_tables::constants::SQLSERVER_ADAPTER; +use crate::state_tables::models::HistoryRead; + +/// Cursor values are already rendered by the adapter; `None` sorts before every SQL value. +pub fn read_history_sql( + qualified_name: &str, + after: Option<&[Option]>, + read: HistoryRead<'_>, +) -> Result { + let HistoryRead { + columns, + timestamp_column, + observed_since, + adapter, + has_event_id, + limit, + } = read; + let mut columns = columns.to_vec(); + let mut projection = columns.join(", "); + projection.push_str(if has_event_id { + ", event_id" + } else { + ", NULL AS event_id" + }); + columns.push(if has_event_id { "event_id" } else { "NULL" }); + let ordered = columns + .iter() + .map(|column| { + if *column == timestamp_column { + Ok((*column).to_owned()) + } else { + binary(column, adapter) + } + }) + .collect::, _>>()?; + let mut greater: Vec = Vec::new(); + let mut equal: Vec = Vec::new(); + if let Some(after) = after { + for ((column, ordered), value) in columns.iter().zip(&ordered).zip(after) { + let literal = value + .as_ref() + .map(|value| { + if *column == timestamp_column { + Ok(value.clone()) + } else { + binary(value, adapter) + } + }) + .transpose()?; + let comparison = literal.as_ref().map_or_else( + || format!("{column} IS NOT NULL"), + |literal| format!("{ordered} > {literal}"), + ); + let prefix = if equal.is_empty() { + String::new() + } else { + format!("{} AND ", equal.join(" AND ")) + }; + greater.push(format!("({prefix}{comparison})")); + equal.push(literal.as_ref().map_or_else( + || format!("{column} IS NULL"), + |literal| format!("{ordered} = {literal}"), + )); + } + } + let mut filters: Vec = Vec::new(); + if !greater.is_empty() { + filters.push(format!("({})", greater.join(" OR "))); + } + if let Some(since) = observed_since { + filters.push(format!("{timestamp_column} >= {since}")); + } + let filter = if filters.is_empty() { + String::new() + } else { + format!(" WHERE {}", filters.join(" AND ")) + }; + let order: Vec = columns + .iter() + .zip(&ordered) + .map(|(column, ordered)| format!("CASE WHEN {column} IS NULL THEN 0 ELSE 1 END, {ordered}")) + .collect(); + let top = adapter == SQLSERVER_ADAPTER; + let head = if top { + format!("TOP {limit} ") + } else { + String::new() + }; + let tail = if top { + String::new() + } else { + format!(" LIMIT {limit}") + }; + Ok(format!( + "SELECT {head}{projection} FROM {qualified_name}{filter} ORDER BY {}{tail}", + order.join(", ") + )) +} + +/// Session and column collations must not merge case, accents or trailing spaces at page boundaries. +fn binary(value: &str, adapter: &str) -> Result { + Ok(match adapter { + "duckdb" | "motherduck" => format!("encode({value})"), + "sqlite" => format!("CAST({value} AS BLOB)"), + "postgres" => format!("convert_to({value}, 'UTF8')"), + "snowflake" => format!("TO_BINARY({value}, 'UTF-8')"), + "bigquery" => format!("CAST({value} AS BYTES)"), + "databricks" => format!("CAST({value} AS BINARY)"), + "sqlserver" => format!("CONVERT(VARBINARY(MAX), CONVERT(NVARCHAR(MAX), {value}))"), + "clickhouse" => value.to_owned(), + _ => { + return Err(format!( + "warehouse history does not support adapter '{adapter}'" + )); + } + }) +} diff --git a/crates/sqlbuild-state/src/state_tables/mod.rs b/crates/sqlbuild-state/src/state_tables/mod.rs index 1a70a48981..042268f15e 100644 --- a/crates/sqlbuild-state/src/state_tables/mod.rs +++ b/crates/sqlbuild-state/src/state_tables/mod.rs @@ -1,3 +1,4 @@ +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-state/src/state_tables/models.rs b/crates/sqlbuild-state/src/state_tables/models.rs index 7a343e134d..8c8a1f5792 100644 --- a/crates/sqlbuild-state/src/state_tables/models.rs +++ b/crates/sqlbuild-state/src/state_tables/models.rs @@ -7,3 +7,14 @@ pub struct TableColumn { pub rendered_type: String, pub required: bool, } + +/// A bounded legacy-compatible history projection, with its optional source event ID. +#[derive(Clone, Copy, Debug)] +pub struct HistoryRead<'a> { + pub columns: &'a [&'a str], + pub timestamp_column: &'a str, + pub observed_since: Option<&'a str>, + pub adapter: &'a str, + pub has_event_id: bool, + pub limit: usize, +} diff --git a/fensu.toml b/fensu.toml index 7e2ae7561a..5a13ca223d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -229,7 +229,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl [[targets.python.dead_code.roots]] modules = ["sqlbuild.compiler.discovery._helpers.filesystem.project_modules"] -symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] +symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_orchestration_module", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] @@ -1564,6 +1564,18 @@ rule = "FPRSS201" path = "crates/sqlbuild-python/src/bindings/models.rs" reason = "CompactAnalysisJob holds a prepared analysis closure, which cannot implement Debug; other models derive it." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/credentials/models.rs" +symbols = ["IssuedToken"] +reason = "IssuedToken implements Debug manually to redact its bearer credential; deriving Debug would expose the secret." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-orchestration/src/platform/models.rs" +symbols = ["PlatformWebhook"] +reason = "PlatformWebhook implements Debug manually to redact its endpoint, which may contain delivery credentials; other models derive Debug." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs" @@ -1617,6 +1629,7 @@ crate_names = [ "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", + "sqlbuild-orchestration", "sqlbuild-project", "sqlbuild-pysource", "sqlbuild-python", @@ -1668,17 +1681,17 @@ edge_paths = [ "crates/sqlbuild-python/src/bindings/_helpers/records/registry.rs", ] pending_budgets = [ - "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=20", + "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", @@ -1693,43 +1706,43 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=22", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", + "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/cli/surrogates.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs=9", @@ -1738,19 +1751,19 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=42", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=41", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs=30", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs=45", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=40", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs=38", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=23", + "crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs=39", "crates/sqlbuild-python/src/bindings/_helpers/compile_context/compile_timings.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compile_context/diagnostic_collector.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/compile_context/input_reads.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs=3", @@ -1778,22 +1791,22 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs=8", @@ -1810,8 +1823,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", @@ -1833,11 +1846,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs=19", @@ -1849,8 +1862,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_connection.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", @@ -1859,7 +1872,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=15", + "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs=4", @@ -1875,7 +1888,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_config.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs=6", @@ -1890,16 +1903,16 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=29", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=17", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=12", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=28", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=28", + "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs=16", @@ -1924,7 +1937,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs=5", @@ -1935,7 +1948,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_calls.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_external.rs=4", @@ -1963,7 +1976,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs=3", @@ -1973,11 +1986,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/module_digests.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs=1", @@ -1988,16 +2001,16 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=33", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/typed_config.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=12", @@ -2017,10 +2030,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs=1", @@ -2040,7 +2053,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs=4", @@ -2054,9 +2067,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=2", @@ -2089,11 +2102,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=15", + "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=1", @@ -2101,12 +2114,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/document.rs=0", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=1", @@ -2118,8 +2131,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs=1", @@ -2130,8 +2143,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs=1", @@ -2146,24 +2159,24 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=17", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=20", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=14", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs=1", @@ -2171,11 +2184,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_observation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs=21", - "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/rules/fix_facts.rs=8", @@ -2190,7 +2203,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=1", @@ -2202,26 +2215,26 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=33", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=32", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=6", @@ -2229,7 +2242,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=101", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=100", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", @@ -2242,8 +2255,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/events.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs=4", @@ -2256,7 +2269,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/type_system/equality.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=8", - "crates/sqlbuild-python/src/bindings/models.rs=7", + "crates/sqlbuild-python/src/bindings/models.rs=0", ] [targets.rust.rule_options.XSB073] @@ -2635,12 +2648,16 @@ module_layers = [ "lineage/lineage_command=commands", "lineage/lineage_facts=project", "load=commands", + "location_command=commands", "macro_bridge=front", "manifest=compile", "microbatches=plan", "model_config=front", "native_host=infra", "native_host/model_lookup=project", + "native_host/orchestration_sensors=commands", + "native_host/orchestration_triggers=project", + "native_host/warehouse_metadata=commands", "native_threads=infra", "output_capture=output", "plan_command=commands", @@ -2735,3 +2752,8 @@ rule = "FFS120" path = "src/sqlbuild/executor/scheduling/classes/recovering_connection_pool.py" symbols = ["RecoveringConnectionPool.put"] reason = "The override keeps queue.Queue.put's positional protocol, which every pool consumer calls." + +[[targets.python.rule_exceptions]] +rule = "FFS201" +path = "src/sqlbuild/orchestration_declarations/models/sensor_context.py" +reason = "Only SensorContext lives here: invocation-local ctx.cursor assignment proposes progress, committed solely by native fenced completion. The package regression guards this declaration-only scope; class-symbol exceptions are unsupported." diff --git a/pyproject.toml b/pyproject.toml index f66e346fcf..daf4fa4c8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.28.0", + "fensu>=0.28.3", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/python-boundary.toml b/python-boundary.toml index ee2346f85d..5829d6322f 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -12,6 +12,31 @@ # data instead of projecting those public containers and immediately reading them back. remaining_lines = 0 +[[stays]] +path = "orchestration.py" +rule = 3 +note = "Public orchestration facade: re-exports only." + +[[stays]] +path = "orchestration_declarations/models/authoring.py" +rule = 3 +note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." + +[[stays]] +path = "orchestration_declarations/models/sensor_context.py" +rule = 3 +note = "Only the mutable SensorContext authoring declaration required by ctx.cursor assignment; no Python runtime allowance. Native fenced completion validates and commits proposed progress." + +[[stays]] +path = "orchestration_declarations/types.py" +rule = 3 +note = "Public PlatformEventKind, RunEventKind and NodeEventKind enums only, placed in types.py to satisfy enum ownership while retaining a zero implementation budget; no Python event processing allowance." + +[[stays]] +path = "orchestration_declarations/main/compile_snapshot.py" +rule = 3 +note = "Public declaration compiler: one-line delegation to build_orchestration_snapshot; no Python discovery, validation or snapshot logic." + [[stays]] path = "executor" rule = 1 diff --git a/scripts/fensu_policy/_helpers/native_crossings.py b/scripts/fensu_policy/_helpers/native_crossings.py index 430cc3fb78..27df706697 100644 --- a/scripts/fensu_policy/_helpers/native_crossings.py +++ b/scripts/fensu_policy/_helpers/native_crossings.py @@ -27,6 +27,11 @@ _PYTHON_FIELD: re.Pattern[str] = re.compile(NATIVE_PYTHON_FIELD_PATTERN) _STRUCT_OPEN: re.Pattern[str] = re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?struct\s+\w+[^;]*\{\s*$") _FIELD: re.Pattern[str] = re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?\w+\s*:") +_PYCLASS_MODULE: re.Pattern[str] = re.compile( + r"(^[ \t]*#\[\s*(?:pyo3::)?pyclass\s*\([^]]*?\bmodule\s*=\s*(?:r#*)?)" + r'"(?:\\.|[^"\\\n])*"', + re.MULTILINE, +) def is_checked_binding(*, path: str) -> bool: @@ -72,8 +77,9 @@ def _code_lines(*, source: str) -> list[tuple[int, str]]: def crossing_lines(*, source: str) -> list[int]: - """Line numbers of Rust-to-Python crossings, one entry per crossing.""" + """Line numbers of Rust-to-Python crossings, excluding inert PyO3 module metadata.""" + source = _PYCLASS_MODULE.sub(r'\1""', source) found: list[int] = [] number: int line: str diff --git a/scripts/native_tests/__init__.py b/scripts/native_tests/__init__.py new file mode 100644 index 0000000000..4175a3327b --- /dev/null +++ b/scripts/native_tests/__init__.py @@ -0,0 +1 @@ +"""Shared native integration-test infrastructure.""" diff --git a/scripts/native_tests/constants.py b/scripts/native_tests/constants.py new file mode 100644 index 0000000000..3473a3cec7 --- /dev/null +++ b/scripts/native_tests/constants.py @@ -0,0 +1,6 @@ +"""Reproducible native integration database image.""" + +POSTGRES_IMAGE: str = ( + "postgres:17.6-alpine3.22@" + "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94" +) diff --git a/scripts/native_tests/main/__init__.py b/scripts/native_tests/main/__init__.py new file mode 100644 index 0000000000..22055e8817 --- /dev/null +++ b/scripts/native_tests/main/__init__.py @@ -0,0 +1 @@ +"""Native test command entrypoints.""" diff --git a/scripts/native_tests/main/run.py b/scripts/native_tests/main/run.py new file mode 100644 index 0000000000..fa3a52fc5e --- /dev/null +++ b/scripts/native_tests/main/run.py @@ -0,0 +1,33 @@ +"""One disposable database per native suite; Rust fixtures isolate their own schemas.""" + +from __future__ import annotations + +import os +import subprocess +import sys + +from testcontainers.postgres import PostgresContainer + +from scripts.native_tests.constants import POSTGRES_IMAGE + + +def run_native_tests(argv: list[str] | None = None) -> int: + """Fail closed on Docker/startup errors and propagate Cargo's result after cleanup.""" + arguments: list[str] = sys.argv[1:] if argv is None else argv + print("Starting disposable PostgreSQL for native integration tests...", flush=True) + try: + with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: + print("PostgreSQL ready. Running native tests with isolated schemas...", flush=True) + result: subprocess.CompletedProcess[bytes] = subprocess.run( + ["cargo", "test", *arguments], + env=os.environ + | { + "SQLBUILD_TEST_CONTROL_POSTGRES_URL": postgres.get_connection_url(), + }, + check=False, + ) + print(f"PostgreSQL removed. Native tests exited {result.returncode}.", flush=True) + return result.returncode + except Exception as error: + print(f"Native integration tests failed: {error}", file=sys.stderr, flush=True) + return 1 diff --git a/scripts/run_native_tests.py b/scripts/run_native_tests.py new file mode 100644 index 0000000000..2f673d1fc7 --- /dev/null +++ b/scripts/run_native_tests.py @@ -0,0 +1,12 @@ +"""Direct wrapper for native tests with disposable PostgreSQL.""" + +from scripts.native_tests.main.run import run_native_tests + + +def main(argv: list[str] | None = None) -> int: + """Run Cargo tests with a shared Testcontainers PostgreSQL instance.""" + return run_native_tests(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4a78cb2ea7..31832558f1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -199,6 +199,7 @@ COLUMN_SCHEMA_FINGERPRINT: str COLUMN_DEFINITION_B64: str COLUMN_METADATA_JSON_B64: str COLUMN_TIMESTAMP: str +COLUMN_EVENT_ID: str NODE_TYPE_MODEL: str NODE_TYPE_SEED: str AUDIT_GATE_METADATA_KEY: str @@ -1738,6 +1739,48 @@ class NativeCompiledAudit: @property def meta(self) -> dict[str, object] | None: ... +class NativeOrchestrationSnapshot: + """Compiled jobs, schedules and sensors retained as data until native publication.""" + + def to_json(self) -> str: ... + def publish(self, server_url: str, bearer: str) -> str: ... + def start_worker( + self, server_url: str, bearer: str, identity: tuple[str, str] + ) -> NativeOrchestrationWorker: ... + +class NativeOrchestrationWorker: + """A verified claim owner with joined native heartbeat teardown.""" + + def execution_json(self) -> str: ... + def cancel_requested(self) -> bool: ... + def report(self, event_id: str, change_json: str) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, + kind: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: ... + +def build_orchestration_snapshot( + project: CompiledProjectView, + metadata_json: str, + project_dir: Path, + *, + declarations: Sequence[object] | None = None, +) -> NativeOrchestrationSnapshot: ... +def run_location_command( + *, + project_dir: Path, + location_id: str, + selected_target: str, + generation: int, + server_url: str, + image_id: str | None = None, + action: str = "publish", +) -> int: ... + class NativeProjectView: @property def run_id(self) -> str: ... @@ -4145,6 +4188,8 @@ def write_fingerprint( render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, ensure_table: bool = True, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> bool: ... def write_fingerprints( *, @@ -4158,6 +4203,8 @@ def write_fingerprints( render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, on_progress: FingerprintWriteProgress | None = None, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> None: ... def select_latest_fingerprints( *, @@ -7270,6 +7317,7 @@ class CliEntrypointHandlers(NativeRecord): run_refactor: Callable[..., int] | None run_recall: Callable[..., int] | None run_review: Callable[..., int] | None + run_location: Callable[..., int] | None def __init__( self, run_compile: Callable[[CompileCommandRequest], int], @@ -7306,6 +7354,7 @@ class CliEntrypointHandlers(NativeRecord): run_refactor: Callable[..., int] | None = None, run_recall: Callable[..., int] | None = None, run_review: Callable[..., int] | None = None, + run_location: Callable[..., int] | None = None, ) -> None: ... class CommandWarehouseScope(NativeRecord): @@ -7794,6 +7843,8 @@ class SourceFreshnessRenderers(NativeRecord): render_insert_records_sql: Callable[..., str] render_create_table_sql: Callable[..., str] | None render_create_index_sqls: Callable[..., tuple[str, ...]] | None + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None + render_add_columns: Callable[..., tuple[str, ...]] | None def __init__( self, render_qualified_name: Callable[..., str | None], @@ -7801,6 +7852,8 @@ class SourceFreshnessRenderers(NativeRecord): render_insert_records_sql: Callable[..., str], render_create_table_sql: Callable[..., str] | None = None, render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, + read_columns: Callable[..., tuple[ColumnInfo, ...]] | None = None, + render_add_columns: Callable[..., tuple[str, ...]] | None = None, ) -> None: ... class SourceFreshnessObservation(NativeRecord): @@ -8021,6 +8074,7 @@ class CliCommand(StrEnum): DIFF = "diff" REVIEW = "review" DEBUG = "debug" + LOCATION = "location" LINEAGE = "lineage" QUERY = "query" COST = "cost" diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 9dca6ce419..fde2109986 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -167,6 +167,10 @@ "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "load_sink_module", ), + "load_orchestration_module": ( + "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", + "load_orchestration_module", + ), "module_functions": ( "sqlbuild.compiler.discovery._helpers.filesystem.project_modules", "module_functions", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index c68e33d507..4ca6064432 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -130,6 +130,9 @@ from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( load_materialize_function as load_materialize_function, ) +from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( + load_orchestration_module as load_orchestration_module, +) from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( load_provider_module as load_provider_module, ) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index d376a6a228..f6d52b95c4 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1080,7 +1080,9 @@ def connect(self, config: dict[str, Any]) -> Any: import duckdb database: str = str(config.get("database", ":memory:")) - raw_connection: duckdb.DuckDBPyConnection = duckdb.connect(database=database) + raw_connection: duckdb.DuckDBPyConnection = duckdb.connect( + database=database, read_only=bool(config.get("read_only", False)) + ) return self._configured_connection(raw_connection=raw_connection, config=config) def connect_local_replay(self, database: Path) -> Any: diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 2f200b531d..25d13ae0f1 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -340,6 +340,7 @@ def render_create_fingerprint_table_sql( from sqlbuild._native import ( COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, + COLUMN_EVENT_ID, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, COLUMN_NODE_TYPE, @@ -373,7 +374,8 @@ def render_create_fingerprint_table_sql( f"{COLUMN_SCHEMA_FINGERPRINT} NVARCHAR(450) NOT NULL, " f"{COLUMN_DEFINITION_B64} NVARCHAR(MAX) NOT NULL, " f"{COLUMN_METADATA_JSON_B64} NVARCHAR(MAX) NOT NULL, " - f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL" + f"{COLUMN_TIMESTAMP} DATETIME2 NOT NULL, " + f"{COLUMN_EVENT_ID} NVARCHAR(MAX)" f")" ) return self._create_table_if_missing_sql( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index f28340d71b..1eef8c4987 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -244,6 +244,17 @@ def load_sink_module(*, file_path: Path, project_dir: Path) -> ModuleType: ) +def load_orchestration_module(*, file_path: Path, project_dir: Path) -> ModuleType: + """Execute an orchestration module with the project's normal package imports.""" + + return _load_project_package_module( + file_path=file_path, + project_dir=project_dir, + file_label="orchestration", + error_type=PythonNodeDiscoveryError, + ) + + def load_materialize_function( *, file_path: Path, project_dir: Path ) -> Callable[..., object] | None: @@ -338,6 +349,8 @@ def _exec_project_module( sys.modules[module_name] = module spec.loader.exec_module(module) except Exception as error: + if sys.modules.get(module_name) is module: + sys.modules.pop(module_name, None) raise error_type( f"Failed to import {file_label} file {file_path.relative_to(project_dir)}: {error}" ) from error diff --git a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py index 5c2abd241e..0378d85973 100644 --- a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py @@ -205,6 +205,8 @@ def _on_group_progress( fingerprints=group, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, on_progress=_on_group_progress, diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index d8d0c9f088..f4559e8027 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -118,6 +118,8 @@ def materialize_query_diff_artifact( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index a95ec0cd3b..c7f935e162 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -182,6 +182,8 @@ def _try_write_function_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py index edc76da34c..310f994670 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py @@ -55,6 +55,8 @@ def try_write_python_node_identity_fingerprint( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/review/_helpers/marker.py b/src/sqlbuild/executor/review/_helpers/marker.py index ee87afbfbf..1f2f23e173 100644 --- a/src/sqlbuild/executor/review/_helpers/marker.py +++ b/src/sqlbuild/executor/review/_helpers/marker.py @@ -52,6 +52,8 @@ def write_review_marker( ), render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index e084b6ab5d..065fe30123 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -97,6 +97,8 @@ def try_write_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index a103b8bb6b..02a44cc063 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -56,6 +56,8 @@ def try_write_seed_fingerprint( fingerprint=fingerprint, render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, render_create_table_sql=adapter.render_create_fingerprint_table_sql, render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, ) diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py new file mode 100644 index 0000000000..568ce36709 --- /dev/null +++ b/src/sqlbuild/orchestration.py @@ -0,0 +1,57 @@ +"""Public authoring and snapshot compilation API for orchestration declarations.""" + +from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule, + CalendarWindow, + Job, + JobGroup, + NodeEventCondition, + PlatformEventCondition, + RunEventCondition, + RunRequest, + Schedule, + Sensor, + SkipReason, +) +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule as calendar_schedule, +) +from sqlbuild.orchestration_declarations.models.authoring import Job as job +from sqlbuild.orchestration_declarations.models.authoring import JobGroup as job_group +from sqlbuild.orchestration_declarations.models.authoring import NodeEventCondition as node_event +from sqlbuild.orchestration_declarations.models.authoring import ( + PlatformEventCondition as platform_event, +) +from sqlbuild.orchestration_declarations.models.authoring import RunEventCondition as run_event +from sqlbuild.orchestration_declarations.models.authoring import Schedule as schedule +from sqlbuild.orchestration_declarations.models.authoring import Sensor as sensor +from sqlbuild.orchestration_declarations.models.sensor_context import SensorContext +from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind + +__all__ = ( + "CalendarSchedule", + "CalendarWindow", + "Job", + "JobGroup", + "NodeEventCondition", + "NodeEventKind", + "PlatformEventCondition", + "PlatformEventKind", + "RunEventCondition", + "RunEventKind", + "RunRequest", + "Schedule", + "Sensor", + "SensorContext", + "SkipReason", + "calendar_schedule", + "compile_snapshot", + "job", + "job_group", + "node_event", + "platform_event", + "run_event", + "schedule", + "sensor", +) diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py new file mode 100644 index 0000000000..0f9fc8b052 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -0,0 +1,28 @@ +"""Delegate authoring-record compilation to the native retained snapshot owner.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +from sqlbuild._native import NativeOrchestrationSnapshot, build_orchestration_snapshot +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.orchestration_declarations.models.authoring import ( + CalendarSchedule, + Job, + JobGroup, + Sensor, +) + + +def compile_snapshot( + *, + project: CompiledProjectView, + metadata_json: str, + project_dir: Path, + declarations: Sequence[Job | JobGroup | CalendarSchedule | Sensor] | None = None, +) -> NativeOrchestrationSnapshot: + """Compile authoring records and retained project facts into a data-only snapshot.""" + return build_orchestration_snapshot( + project, metadata_json, project_dir, declarations=declarations + ) diff --git a/src/sqlbuild/orchestration_declarations/models/__init__.py b/src/sqlbuild/orchestration_declarations/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py new file mode 100644 index 0000000000..fd188208ed --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -0,0 +1,144 @@ +"""Authoring records only; selection, validation and scheduling belong to Rust.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass, field, replace +from datetime import datetime +from typing import Literal + +from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind + + +@dataclass(frozen=True, kw_only=True) +class Schedule: + """Cron with an RFC 3339 start; catch_up='all' requires a positive max_catch_up.""" + + cron: str + timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + catch_up: Literal["latest", "all"] = "latest" + max_catch_up: int | None = None + + +@dataclass(frozen=True, eq=False) +class Job: + """A named selection to execute, optionally on a wall-clock schedule.""" + + name: str + description: str = field(kw_only=True) + select: str = field(kw_only=True) + exclude: str = field(default="", kw_only=True) + overlap: Literal["coalesce", "queue", "allow"] = field(default="coalesce", kw_only=True) + on_failure: Literal["skip_dependents", "skip_downstream_jobs", "stop_group"] = field( + default="skip_dependents", kw_only=True + ) + settings: dict[str, object] = field(default_factory=dict, kw_only=True) + concurrency: int | None = field(default=None, kw_only=True) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) + + +@dataclass(frozen=True, eq=False) +class JobGroup: + """Jobs ordered by compiled lineage and optional explicit dependencies.""" + + name: str + description: str = field(kw_only=True) + jobs: Sequence[Job] = field(kw_only=True) + after: dict[Job, Sequence[Job]] = field(default_factory=dict, kw_only=True) + overlap: Literal["coalesce", "queue", "allow"] = field(default="coalesce", kw_only=True) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) + + +@dataclass(frozen=True, kw_only=True) +class CalendarWindow: + """Timezone-aware, half-open interval for an upcoming calendar evaluation.""" + + start: datetime + end: datetime + + +@dataclass(frozen=True, kw_only=True, eq=False) +class CalendarSchedule: + """Decorate a location callback returning upcoming timezone-aware datetimes.""" + + job: Job | JobGroup + description: str + horizon: str + timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + catch_up: Literal["latest", "all"] = "latest" + max_catch_up: int | None = None + targets: tuple[str, ...] = () + function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) + + def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: + return replace(self, function=function) + + +@dataclass(frozen=True, kw_only=True) +class PlatformEventCondition: + """Consume matching server observations in durable storage order.""" + + kinds: Sequence[PlatformEventKind] + kind: Literal["platform_event"] = field(default="platform_event", init=False) + + +@dataclass(frozen=True, kw_only=True) +class RunEventCondition: + """Match scoped terminal outcomes, optionally requiring genuinely executed successful nodes.""" + + job: Job | JobGroup + kinds: Sequence[RunEventKind] + executed_successfully: Sequence[str] | None = None + kind: Literal["run_event"] = field(default="run_event", init=False) + + +@dataclass(frozen=True, kw_only=True) +class NodeEventCondition: + """Match each successful rebuild in any run within the sensor's location and target.""" + + select: str + kinds: Sequence[NodeEventKind] + kind: Literal["node_event"] = field(default="node_event", init=False) + + +@dataclass(frozen=True, eq=False) +class Sensor: + """Start a job for each matching event; event identity deduplicates replayed requests.""" + + name: str + description: str = field(kw_only=True) + when: PlatformEventCondition | RunEventCondition | NodeEventCondition | None = field( + default=None, kw_only=True + ) + run: Job | JobGroup = field(kw_only=True) + minimum_interval_seconds: int = field(default=30, kw_only=True) + default_status: Literal["running", "stopped"] = field(default="running", kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) + function: Callable[..., Iterable[RunRequest] | SkipReason] | None = field( + default=None, repr=False, kw_only=True + ) + + def __call__(self, function: Callable[..., Iterable[RunRequest] | SkipReason]) -> Sensor: + return replace(self, function=function) + + +@dataclass(frozen=True, kw_only=True) +class RunRequest: + """A sensor-local key deduplicates requests across evaluations and retries.""" + + run_key: str + + +@dataclass(frozen=True) +class SkipReason: + """Complete an evaluation without starting a run.""" + + reason: str diff --git a/src/sqlbuild/orchestration_declarations/models/sensor_context.py b/src/sqlbuild/orchestration_declarations/models/sensor_context.py new file mode 100644 index 0000000000..648ce981db --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/models/sensor_context.py @@ -0,0 +1,12 @@ +"""Invocation-local context; native fenced completion alone commits a proposed cursor.""" + +from dataclasses import dataclass + + +@dataclass(kw_only=True) +class SensorContext: + """Cursor assignment proposes progress; failure or timeout retains the committed cursor.""" + + cursor: str | None + target: str + evaluation_id: str diff --git a/src/sqlbuild/orchestration_declarations/types.py b/src/sqlbuild/orchestration_declarations/types.py new file mode 100644 index 0000000000..4635074185 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/types.py @@ -0,0 +1,32 @@ +"""Public enum vocabulary only; event processing belongs to Rust.""" + +from enum import StrEnum + + +class PlatformEventKind(StrEnum): + """Launch failures include response errors and do not prove worker absence.""" + + HEARTBEAT = "heartbeat" + SCHEDULER_NO_LEADER = "scheduler_no_leader" + TICK_OVERDUE = "tick_overdue" + CALENDAR_HORIZON_LOW = "calendar_horizon_low" + LOCATION_BROKEN = "location_broken" + LAUNCH_FAILED = "launch_failed" + + +class RunEventKind(StrEnum): + """COMPLETED selects every terminal outcome; it is not a run status.""" + + COMPLETED = "completed" + SUCCESS = "success" + WARN = "warn" + PARTIAL = "partial" + FAILED = "failed" + CANCELLED = "cancelled" + PRESUMED_FAILED = "presumed_failed" + + +class NodeEventKind(StrEnum): + """SUCCESS requires a genuine rebuild, not freshness-only completion.""" + + SUCCESS = "success" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py index 4fe9306891..ebb83d9c96 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py @@ -146,7 +146,10 @@ def test_given_unchanged_project_when_planning_after_build_then_models_are_not_q try: stale_row: tuple[object, ...] = tuple(fingerprint_rows[0]) connection.execute( - "INSERT INTO main._sqlbuild_fingerprints VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO main._sqlbuild_fingerprints " + "(node_type, node_name, target_database, target_schema, target_name, run_id, " + "definition_hash, version_hash, schema_fingerprint, definition_b64, " + "metadata_json_b64, ts) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( stale_row[0], stale_row[1], diff --git a/tests/integration/scripts/fensu_policy/__init__.py b/tests/integration/scripts/fensu_policy/__init__.py new file mode 100644 index 0000000000..6aab21b701 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/__init__.py @@ -0,0 +1 @@ +"""Integration checks for repository-authored Fensu policies.""" diff --git a/tests/integration/scripts/fensu_policy/_test_types.py b/tests/integration/scripts/fensu_policy/_test_types.py new file mode 100644 index 0000000000..ef287d1b65 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/_test_types.py @@ -0,0 +1,10 @@ +"""Native boundary metadata cases for the real Fensu CLI.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeBoundaryTestCase: + description: str + metadata: str + expected_fault_count: int diff --git a/tests/integration/scripts/fensu_policy/test_native_boundary.py b/tests/integration/scripts/fensu_policy/test_native_boundary.py new file mode 100644 index 0000000000..1b996fae04 --- /dev/null +++ b/tests/integration/scripts/fensu_policy/test_native_boundary.py @@ -0,0 +1,78 @@ +"""Exercise native boundary policy through Fensu's real Rust analyzer and rule host.""" + +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.integration.scripts.fensu_policy._test_types import NativeBoundaryTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + NativeBoundaryTestCase( + description="pyclass module metadata", + metadata='#[pyclass(frozen, module = "sqlbuild._native")]\npub struct Snapshot;\n', + expected_fault_count=2, + ), + NativeBoundaryTestCase( + description="namespaced pyclass raw module metadata", + metadata='#[pyo3::pyclass(frozen, module = r#"sqlbuild._native"#)]\npub struct Snapshot;\n', + expected_fault_count=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_pyclass_metadata_when_checking_rust_then_only_real_python_crossings_fault( + tmp_path: Path, + test_case: NativeBoundaryTestCase, +) -> None: + repository: Path = Path(__file__).resolve().parents[4] + shutil.copytree( + repository / "scripts/fensu_policy", + tmp_path / "scripts/fensu_policy", + ignore=shutil.ignore_patterns("__pycache__"), + ) + (tmp_path / "scripts/__init__.py").write_text("") + (tmp_path / "Cargo.toml").write_text( + '[workspace]\nmembers = ["crates/sqlbuild-python"]\nresolver = "2"\n' + ) + crate: Path = tmp_path / "crates/sqlbuild-python" + (crate / "src").mkdir(parents=True) + (crate / "Cargo.toml").write_text( + '[package]\nname = "sqlbuild-python"\nversion = "0.1.0"\nedition = "2021"\n' + ) + (tmp_path / "fensu.toml").write_text( + '[targets.rust]\nanalyzer = "rust"\nroots = ["crates"]\n' + 'rule_modules = ["scripts.fensu_policy.rules.native_boundary"]\n' + 'select = ["XSB072"]\n' + ) + source: Path = crate / "src/lib.rs" + source.write_text(test_case.metadata) + command: list[str] = [ + str(Path(sys.executable).with_name("fensu")), + "check", + "--target", + "rust", + "--no-cache", + "--no-color", + ] + clean: subprocess.CompletedProcess[str] = subprocess.run( + command, cwd=tmp_path, capture_output=True, text=True, timeout=15, check=False + ) + assert clean.returncode == 0, clean.stdout + clean.stderr + assert "Found 0 faults" in clean.stdout + source.write_text( + test_case.metadata + 'fn read(py: Python<\'_>) { py.import("sqlbuild.compiler"); }\n' + ) + crossing: subprocess.CompletedProcess[str] = subprocess.run( + command, cwd=tmp_path, capture_output=True, text=True, timeout=15, check=False + ) + assert crossing.returncode == 1, crossing.stdout + crossing.stderr + assert crossing.stdout.count("XSB072") == test_case.expected_fault_count, crossing.stdout + assert crossing.stdout.count(" --> crates/sqlbuild-python/src/lib.rs:3:0") == 2 + assert " --> crates/sqlbuild-python/src/lib.rs:1:0" not in crossing.stdout + assert source.read_text().startswith(test_case.metadata) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index bfb3ca0eab..c3169eea69 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -4,6 +4,8 @@ from dataclasses import dataclass, field +from sqlbuild.orchestration import CalendarSchedule, Job, JobGroup, Schedule, Sensor + @dataclass(frozen=True) class HeaderProjectionTestCase: @@ -190,3 +192,74 @@ class QuotedHookBoundaryTestCase: description: str argument: str expected_sql: str + + +@dataclass(frozen=True) +class SnapshotCatalogTestCase: + description: str + files: dict[str, str] + expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class SnapshotRejectionTestCase: + description: str + files: dict[str, str] + nodes: dict[str, object] + expected_compiler_errors: bool + expected_message: str + + +@dataclass(frozen=True) +class SnapshotEnvironmentTestCase: + description: str + first_lockfile: str + second_lockfile: str + expected_environment_count: int + + +@dataclass(frozen=True) +class SnapshotScheduleTestCase: + description: str + schedule: Schedule + expected_selection: tuple[str, ...] + expected_controls: dict[str, object] = field( + default_factory=lambda: { + "starts_at": None, + "default_status": "running", + "catch_up": "latest", + } + ) + + +@dataclass(frozen=True) +class SnapshotDeclarationRejectionTestCase: + description: str + job_description: str = "Orders." + select: str = "orders" + cron: str = "0 2 * * *" + timezone: str = "UTC" + settings: dict[str, object] = field(default_factory=dict) + concurrency: int | None = None + copies: int = 1 + expected_message: str = "" + schedule: Schedule | None = None + + +@dataclass(frozen=True) +class SnapshotDiscoveryTestCase: + description: str + files: dict[str, str] + expected_jobs: tuple[str, ...] + declarations: tuple[Job | JobGroup | CalendarSchedule | Sensor, ...] | None = None + expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) + expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) + expected_schedules: tuple[str, ...] = () + + +@dataclass(frozen=True) +class SnapshotDiscoveryRejectionTestCase: + description: str + files: dict[str, str] + expected_error: type[Exception] + expected_message: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index f351592f4d..024306d0a4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -8,7 +8,8 @@ from collections.abc import Callable, Iterator from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any +from uuid import UUID import pytest @@ -47,6 +48,20 @@ if TYPE_CHECKING: from sqlbuild._native import CompileInputsView +METADATA: dict[str, Any] = { + "location_id": str(UUID(int=1)), + "target": "development", + "generation": 1, + "git_revision": None, + "git_dirty": False, + "git_author": None, + "image_id": None, + "environment_hash": "test-environment", + "sqb_version": "not-the-installed-version", + "nodes": {}, + "documents": {}, +} + TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py new file mode 100644 index 0000000000..75debee041 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/__init__.py @@ -0,0 +1 @@ +"""Retained compiler snapshots published through the real native control API.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py new file mode 100644 index 0000000000..20b540641f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -0,0 +1,168 @@ +"""Publication cases and isolated server credentials.""" + +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from subprocess import Popen +from typing import Any, Literal, NamedTuple + + +class SnapshotApi(NamedTuple): + url: str + location_id: str + bearer: str + + +class ClaimedWorker(NamedTuple): + run_id: str + claim_id: str + bearer: str + + +class ScheduledWorker(NamedTuple): + api: SnapshotApi + project: Path + control_store: Path + process: Popen[bytes] + log: Path + docker: "DockerLocation | None" = None + + +class DockerLocation(NamedTuple): + image_id: str + container: str + socket: Path + + +class WorkerDeployment(NamedTuple): + kind: str + configuration: dict[str, Any] + publication_arguments: tuple[str, ...] = () + docker: DockerLocation | None = None + + +@dataclass(frozen=True) +class SnapshotPublicationTestCase: + description: str + target: str + expected_generation: int + expected_nodes: frozenset[str] + + +@dataclass(frozen=True) +class SnapshotWorkerTestCase: + description: str + model_file: str + original_sql: str + replacement_sql: str + expected_error: str + + +@dataclass(frozen=True) +class LocationReloadTestCase: + description: str + target: str + expected_generation: int + expected_stop_code: int + database: str + expected_connection: bool + + +@dataclass(frozen=True) +class LocationShutdownTestCase: + description: str + signal: int + expected_exit_code: int + minimum_heartbeats: int + blocked_command: str + + +@dataclass(frozen=True) +class LocationDatabaseTestCase: + description: str + database: str + expected_generation: int + connection_toml: str + + +@dataclass(frozen=True) +class LocationWorkTestCase: + description: str + database: str + expected_connection: bool + + +@dataclass(frozen=True) +class WorkerExecutionTestCase: + description: str + model_sql: str + warehouse_query: str + expected_status: str + expected_rows: list[tuple[int]] + expected_node_count: int = 1 + expected_origin: Literal["schedule", "sensor"] = "schedule" + files: dict[str, str] = field(default_factory=dict) + files_after_publication: dict[str, str] = field(default_factory=dict) + store_setup: str = "" + concurrency_limits: dict[str, int] = field(default_factory=dict) + launcher: Literal["native", "uncooperative", "docker"] = "native" + + +@dataclass(frozen=True) +class GroupExecutionTestCase(WorkerExecutionTestCase): + description: str + on_failure: Literal["skip_dependents", "skip_downstream_jobs", "stop_group"] = "skip_dependents" + expected_child_statuses: dict[str, tuple[str, ...]] = field(default_factory=dict) + expected_totals: tuple[str, str | None] = ("skipped", "model:broken_orders") + expected_marts_started: bool = True + + +@dataclass(frozen=True) +class WorkerRestartTestCase(WorkerExecutionTestCase): + description: str + shutdown_signal: int = 15 + expected_scheduler_exit: int = 0 + + +@dataclass(frozen=True) +class BackfillExecutionTestCase(WorkerExecutionTestCase): + description: str + window_arguments: tuple[str, ...] = () + expected_window: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CalendarRejectionTestCase: + description: str + expression: str + expected_message: str + horizon: str = "8s" + timezone: str = "UTC" + + +@dataclass(frozen=True) +class CalendarRefreshTestCase: + description: str + expected_timezone: str + + +@dataclass(frozen=True) +class WarehouseHistoryTestCase: + description: str + adapter_name: str + expected_page_lengths: tuple[int, ...] + connection_key: str + journal_modes: tuple[str, ...] + append_fingerprints: Callable[..., None] + + +@dataclass(frozen=True) +class FingerprintHistoryTestCase: + description: str + adapter_name: str + connection_key: str + observed_at: datetime + journal_modes: tuple[str, ...] + fixture_paths: tuple[Path, ...] + expected_legacy_runs: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py new file mode 100644 index 0000000000..8d7411a51a --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/__init__.py @@ -0,0 +1 @@ +"""Real container launcher verification.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py new file mode 100644 index 0000000000..36d2ea7541 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/helpers.py @@ -0,0 +1,194 @@ +"""A real Docker location with the installed runtime and an isolated synthetic warehouse.""" + +import json +import os +import shutil +import socket +import subprocess +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from http.client import HTTPConnection, HTTPResponse +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from socketserver import UnixStreamServer +from threading import Event, Thread +from uuid import uuid4 + +import pytest + +import sqlbuild +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + DockerLocation, + WorkerDeployment, +) + +DOCKER_SOCKET: Path = Path(os.environ.get("SQLBUILD_TEST_DOCKER_SOCKET", "/var/run/docker.sock")) +REQUIRES_DOCKER: pytest.MarkDecorator = pytest.mark.skipif( + os.environ.get("SQLBUILD_REQUIRE_DOCKER_TESTS") != "1" + and (sys.platform != "linux" or shutil.which("docker") is None or not DOCKER_SOCKET.exists()), + reason="Docker worker tests require a Linux Docker daemon", +) + + +def docker_command(socket: Path, *arguments: str) -> str: + """Keep daemon errors useful without dumping a container's credential-bearing environment.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + ["docker", "--host", f"unix://{socket}", *arguments], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=20, + check=False, + ) + assert result.returncode == 0, result.stdout + return result.stdout.strip() + + +@contextmanager +def docker_deployment(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + """The worker inherits only these runtime/project mounts, never the store or daemon socket.""" + socket: Path = DOCKER_SOCKET + project: Path = directory / "project" + image: str = os.environ.get("SQLBUILD_TEST_WORKER_IMAGE", "alpine:3.23") + image_id: str = docker_command(socket, "image", "inspect", "--format", "{{.Id}}", image) + name: str = f"sqlbuild-test-location-{uuid4()}" + python: Path = Path(sys.executable) + installations: list[Path] = [Path(sys.prefix), Path(sys.base_prefix)] + while python.is_symlink(): + python = python.parent / python.readlink() + installations.append(python.parent.parent) + mounts: list[str] = [] + for path in filter( + Path.exists, + dict.fromkeys( + [ + Path("/usr"), + Path("/lib"), + Path("/lib64"), + *installations, + Path(sqlbuild.__file__).parent.parent, + ] + ), + ): + mounts.extend(["--volume", f"{path}:{path}:ro"]) + mounts.extend(["--volume", f"{project}:{project}:rw"]) + container: str = docker_command( + socket, + "run", + "--detach", + "--name", + name, + "--network", + "host", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--workdir", + str(project), + "--env", + "HOME=/tmp", + "--env", + "SQLBUILD_WORKER_TEST_VALUE=orders-context", + "--env", + "SQLBUILD_LOCATION_TOKEN=synthetic-location-credential", + "--env", + "SQLBUILD_CONTROL_DATABASE_URL=synthetic-control-store", + "--env", + "SQLBUILD_PLATFORM_WEBHOOK_URL=http://127.0.0.1:1/events?token=synthetic-delivery-credential", + "--env", + "SQLBUILD_SCHEDULER_TEST_STORE=synthetic-configured-store", + *mounts, + image_id, + "/usr/bin/sleep", + "300", + ) + try: + yield WorkerDeployment( + "docker", + {"socket": str(socket), "location_container": container, "executable": executable}, + ("--image-id", image_id), + DockerLocation(image_id, container, socket), + ) + finally: + containers: list[str] = docker_command( + socket, "ps", "--all", "--quiet", "--filter", f"volume={project}" + ).split() + for owned in containers: + _ = (directory / f"docker-{owned}.log").write_text( + docker_command(socket, "logs", owned) + ) + _ = docker_command(socket, "rm", "--force", owned) + + +def worker_container(location: DockerLocation, location_id: str, run_id: str) -> dict[str, object]: + """Return only physical identity/exit evidence, not Docker's secret-bearing configuration.""" + name: str = f"sqlbuild-{location_id}-{run_id}" + result: dict[str, object] = json.loads( + docker_command(location.socket, "inspect", "--format", "{{json .State}}", name) + ) + assert ( + docker_command(location.socket, "inspect", "--format", "{{.Image}}", name) + == location.image_id + ) + return result + + +@contextmanager +def lost_start_response(directory: Path, upstream: Path, inspection: Event) -> Iterator[Path]: + """Fail the real start result and hold physical inspection until explicitly restored.""" + proxy: Path = directory / "docker-proxy.sock" + + class Handler(BaseHTTPRequestHandler): + def forward(self) -> None: + connection: HTTPConnection = HTTPConnection("localhost", timeout=5) + connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.sock.settimeout(5) + connection.sock.connect(str(upstream)) + try: + body: bytes = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + connection.request( + self.command, self.path, body, {"Content-Type": "application/json"} + ) + response: HTTPResponse = connection.getresponse() + payload: bytes = response.read() + status: int = response.status + started: bool = self.command == "POST" and self.path.endswith("/start") + unavailable: bool = ( + self.command == "GET" + and self.path.startswith("/containers/sqlbuild-") + and not inspection.is_set() + ) + assert not started or status == 204, ( + "The actual Docker start must succeed before faulting" + ) + status, payload = { + (False, False): (status, payload), + (True, False): (503, b'{"message":"synthetic lost start response"}'), + (False, True): (503, b'{"message":"synthetic inspection outage"}'), + }[(started, unavailable)] + self.send_response(status) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + finally: + connection.close() + + def do_GET(self) -> None: + self.forward() + + def do_POST(self) -> None: + self.forward() + + def log_message(self, format: str, *args: object) -> None: + """Never copy credential-bearing Docker requests to test output.""" + + with UnixStreamServer(str(proxy), Handler) as server: + thread: Thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield proxy + finally: + server.shutdown() + thread.join(timeout=6) + assert not thread.is_alive(), "Docker fault proxy did not stop" + proxy.unlink(missing_ok=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py new file mode 100644 index 0000000000..ae1526b8c8 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -0,0 +1,710 @@ +"""A standalone native API and real SQLite store, with test-only credential provisioning.""" + +from __future__ import annotations + +import hashlib +import json +import os +import secrets +import signal +import sqlite3 +import subprocess +import sys +import time +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, closing, contextmanager, nullcontext, suppress +from datetime import UTC, datetime, timedelta +from io import StringIO +from pathlib import Path +from typing import Any, cast +from urllib.error import HTTPError +from urllib.parse import quote +from urllib.request import Request, urlopen +from uuid import uuid4 + +import duckdb +import pytest + +from sqlbuild._native import Fingerprint, write_fingerprint +from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ClaimedWorker, + ScheduledWorker, + SnapshotApi, + WorkerDeployment, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( + docker_deployment, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +class RejectedPublicationOutput(StringIO): + """A caller loses only the result stream, after the real API accepts a publication.""" + + def write(self, text: str) -> int: + raise OSError("synthetic publication result delivery failure") + + +@contextmanager +def location_process( + arguments: list[str], log: Path, publications: Path, *, env: dict[str, str] | None = None +) -> Iterator[subprocess.Popen[bytes]]: + """Own and reap the supervisor even when an assertion interrupts a reload.""" + with log.open("ab") as stderr, publications.open("ab") as stdout: + process: subprocess.Popen[bytes] = subprocess.Popen( + arguments, stderr=stderr, stdout=stdout, env=env + ) + try: + yield process + finally: + process.terminate() + try: + _ = process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + _ = process.wait(timeout=5) + + +def open_legacy_warehouse(project: Path, adapter_name: str, connection_key: str) -> tuple[Any, Any]: + """The caller owns the returned real connection; declarations must never run during reads.""" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + f'name = "orders"\nadapter = "{adapter_name}"\n' + f'[connections.local]\n{connection_key} = "warehouse.db"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + }, + ) + adapter: Any = resolve_adapter(adapter_name=adapter_name) + connection: Any = adapter.connect({connection_key: str(project / "warehouse.db")}) + legacy_sql: str = Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_state.sql" + ).read_text() + for statement in legacy_sql.rstrip().removesuffix(";").split(";"): + _ = adapter.execute(connection=connection, sql=statement) + return adapter, connection + + +def warehouse_history_page( + project: Path, + table: str, + after: list[str | None] | None = None, + observed_since: str | None = None, +) -> Any: + """Exercise the real isolated reader without a server, state database or declaration import.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "runtime", + "--server", + "http://127.0.0.1:1", + "--location-id", + "00000000-0000-0000-0000-000000000001", + "--target", + "development", + ], + input=json.dumps( + { + table: { + "location": {"database": None, "schema": "main"}, + "after": after, + "observed_since": observed_since, + } + } + ) + + "\n", + text=True, + capture_output=True, + timeout=20, + env=os.environ | {"SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open"}, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +def wait_for_generation( + api: SnapshotApi, target: str, generation: int, process: subprocess.Popen[bytes], log: Path +) -> dict[str, Any]: + deadline: float = time.monotonic() + 15 + current: dict[str, Any] = {"record": {"generation": 0}} + while current["record"]["generation"] < generation: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, ( + f"Generation {generation} was not published: {log.read_text()}" + ) + try: + current = current_snapshot(api, target) + except HTTPError as error: + assert error.code == 404 + time.sleep(0.02) + assert current["record"]["generation"] == generation, log.read_text() + return current + + +def wait_for_log( + process: subprocess.Popen[bytes], log: Path, text: str, *, timeout: float = 15 +) -> None: + deadline: float = time.monotonic() + timeout + while text not in log.read_text(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + + +def wait_for_location_health( + api: SnapshotApi, + target: str, + health: dict[str, Any], + process: subprocess.Popen[bytes], + log: Path, + *, + minimum_reports: int = 1, +) -> dict[str, Any]: + deadline: float = time.monotonic() + 15 + 30 * (minimum_reports - 1) + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/health", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + key: tuple[str, str] = (target, json.dumps(health, sort_keys=True)) + observations: dict[tuple[str, str], dict[str, dict[str, Any]]] = {} + while len(observations.get(key, ())) < minimum_reports: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + reports: list[dict[str, Any]] = json.load(response) + for report in reversed(reports): + event_id: str = report["report"]["event_id"] + observed_key: tuple[str, str] = ( + report["report"]["target"], + json.dumps(report["report"]["health"], sort_keys=True), + ) + observations.setdefault(observed_key, {})[event_id] = report + time.sleep(0.02) + return list(observations[key].values())[-1] + + +def wait_for_interpreter(process: subprocess.Popen[bytes], pidfile: Path, log: Path) -> int: + deadline: float = time.monotonic() + 15 + while not pidfile.exists(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + return int(pidfile.read_text()) + + +def wait_for_location_work( + api: SnapshotApi, request_id: str, process: subprocess.Popen[bytes], log: Path +) -> dict[str, Any]: + """Observe completion through the scoped API rather than private coordination tables.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/work/{request_id}", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + deadline: float = time.monotonic() + 15 + result: dict[str, Any] = {"result_json": None} + while result["result_json"] is None: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + result = json.load(response) + time.sleep(0.02) + return result + + +def api_address(process: subprocess.Popen[bytes], log: Path) -> str: + """Wait for bounded native startup, never publishing an unready address.""" + deadline: float = time.monotonic() + 10 + while "Control API ready: " not in log.read_text(): + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, "Native API did not become ready" + time.sleep(0.02) + return "http://" + log.read_text().partition("Control API ready: ")[2].splitlines()[0] + + +@contextmanager +def snapshot_api(directory: Path) -> Iterator[SnapshotApi]: + """Only this fixture seeds the control database; the code location uses HTTP exclusively.""" + directory.mkdir() + log: Path = directory / "server.log" + environment: dict[str, str] = os.environ.copy() + environment.pop("SQLBUILD_CONTROL_DATABASE_URL", None) + environment.pop("SQLBUILD_PLATFORM_WEBHOOK_URL", None) + with log.open("wb") as stderr: + process: subprocess.Popen[bytes] = subprocess.Popen( + [str(Path(sys.executable).with_name("sqb")), "server", "--port", "0", "--no-scheduler"], + cwd=directory, + env=environment, + stdout=subprocess.DEVNULL, + stderr=stderr, + ) + try: + url: str = api_address(process, log) + location_id: str = str(uuid4()) + token_id: str = str(uuid4()) + bearer: str = f"{token_id}.{secrets.token_hex(32)}" + now: datetime = datetime.now(UTC) + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_locations (location_id,name,created_by,created_at) " + "VALUES (?, 'orders', 'test-admin', ?)", + (location_id, now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_access_tokens " + "(token_id,secret_hash,location_id,scope_json,created_at,expires_at) " + "VALUES (?,?,?,?,?,?)", + ( + token_id, + hashlib.sha256(bearer.encode()).hexdigest(), + location_id, + json.dumps({"kind": "location", "location_id": location_id}), + now.isoformat(), + (now + timedelta(hours=1)).isoformat(), + ), + ) + yield SnapshotApi(url=url, location_id=location_id, bearer=bearer) + finally: + process.terminate() + try: + _ = process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + _ = process.wait(timeout=5) + + +def current_snapshot(api: SnapshotApi, target: str) -> dict[str, Any]: + """Observe the public API instead of inspecting stored snapshot implementation details.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/{quote(target, safe='')}/snapshots", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + result: dict[str, Any] = json.load(response) + return result + + +def native_launcher(directory: Path, executable: str) -> AbstractContextManager[WorkerDeployment]: + """Use the installed native launcher without a synthetic child.""" + return nullcontext( + WorkerDeployment( + "local", {"project_dir": str(directory / "project"), "executable": executable} + ) + ) + + +@contextmanager +def uncooperative_launcher(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + """Own a synthetic process that ignores TERM, including cleanup after a failed assertion.""" + del executable + script: Path = directory / "worker" + _ = script.write_text( + f"#!{sys.executable}\nimport os, signal, time\nfrom pathlib import Path\n" + "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + "Path('worker.pid').write_text(str(os.getpid()))\n" + "time.sleep(90)\n" + ) + script.chmod(0o700) + try: + yield WorkerDeployment( + "local", {"project_dir": str(directory / "project"), "executable": str(script)} + ) + finally: + with suppress(FileNotFoundError, ProcessLookupError): + os.kill(int((directory / "project/worker.pid").read_text()), signal.SIGKILL) + + +@contextmanager +def scheduled_worker( + directory: Path, monkeypatch: pytest.MonkeyPatch, test_case: WorkerExecutionTestCase +) -> Iterator[ScheduledWorker]: + """Declare, publish and schedule isolated DuckDB work without provisioning claims.""" + project: Path = directory / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n" + test_case.model_sql + "\n" + ), + "orchestration/orders.py": ( + "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" + " assert 'SQLBUILD_PLATFORM_WEBHOOK_URL' not in os.environ\n" + 'annual_orders = job("annual_orders", select="orders", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + } + | test_case.files, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + launchers: dict[str, Callable[[Path, str], AbstractContextManager[WorkerDeployment]]] = { + "native": native_launcher, + "docker": docker_deployment, + "uncooperative": uncooperative_launcher, + } + with ( + snapshot_api(directory / "control") as api, + launchers[test_case.launcher](directory, executable) as deployment, + ): + published: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "location", + "publish", + "--project-dir", + str(project), + "--location-id", + api.location_id, + "--target", + "development", + "--server", + api.url, + "--generation", + "1", + *deployment.publication_arguments, + ], + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert published.returncode == 0, published.stderr + write_files(project_dir=project, files=test_case.files_after_publication) + configuration: Path = directory / "launchers.json" + launcher: dict[str, Any] = { + "location_id": api.location_id, + "target": "development", + "server_url": api.url, + "max_workers": 1, + } | deployment.configuration + _ = configuration.write_text( + json.dumps( + { + deployment.kind: [launcher], + "concurrency_limits": test_case.concurrency_limits, + } + ) + ) + control_store: Path = directory / "control/.sqlbuild/control.sqlite3" + with sqlite3.connect(control_store) as control: + _ = control.executescript(test_case.store_setup) + monkeypatch.setenv("SQLBUILD_SCHEDULER_TEST_STORE", f"sqlite://{control_store}") + monkeypatch.setenv( + "SQLBUILD_PLATFORM_WEBHOOK_URL", + "http://127.0.0.1:1/events?token=synthetic-delivery-credential", + ) + log: Path = directory / "scheduler.log" + with location_process( + [ + executable, + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(configuration), + ], + log, + directory / "scheduler.stdout", + ) as process: + yield ScheduledWorker(api, project, control_store, process, log, deployment.docker) + + +def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) -> dict[str, Any]: + """Run/node outcomes, Recall, claims and the physical warehouse agree.""" + api, project, control_store, process, log, _ = worker + run: dict[str, Any] = wait_for_run(api, process, log) + assert run["node_counts_json"] == {test_case.expected_status: test_case.expected_node_count}, ( + log.read_text() + ) + assert run["job_key"] == "annual_orders" + runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" + headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} + with urlopen( + Request(f"{runs_url}/{run['run_id']}/nodes", headers=headers), timeout=5 + ) as response: + nodes: list[dict[str, Any]] = json.load(response) + assert nodes[0]["node_key"] == "model:orders" + assert nodes[0]["started_at"] is not None + assert nodes[0]["ended_at"] is not None + with sqlite3.connect(control_store) as control: + assert control.execute( + "SELECT origin FROM sqb_runs WHERE run_id=?", (run["run_id"],) + ).fetchone() == (test_case.expected_origin,) + assert control.execute( + "SELECT count(*) FROM recall_events " + "WHERE invocation_id=? AND event_type='run_launched'", + (f"scheduler/{run['run_id']}",), + ).fetchone() == (1,) + contexts: list[tuple[str | None]] = control.execute( + "SELECT external_context FROM recall_invocations WHERE command = 'location'" + ).fetchall() + assert contexts, log.read_text() + assert all(context is not None for (context,) in contexts) + expected_run_ids: set[str] = { + run_id + for (run_id,) in control.execute( + "SELECT run_id FROM sqb_runs WHERE kind<>'group' AND ended_at IS NOT NULL" + ).fetchall() + } + assert { + json.loads(cast(str, context))["run_id"] for (context,) in contexts + } == expected_run_ids + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + return run + + +def running_statement(worker: ScheduledWorker, *, job_key: str | None = None) -> dict[str, Any]: + """The worker has started the real long query, not merely entered its Python interpreter.""" + active: dict[str, Any] = wait_for_run( + worker.api, + worker.process, + worker.log, + pending=frozenset({"queued", "launched"}), + job_key=job_key, + ) + assert active["status"] == "running", worker.log.read_text() + deadline: float = time.monotonic() + 15 + with sqlite3.connect(worker.control_store) as control: + while not control.execute( + "SELECT 1 FROM recall_artifacts WHERE CAST(content AS TEXT) LIKE ?", + ("%range(10000000000)%",), + ).fetchone(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + return active + + +def cancel_running_statement( + worker: ScheduledWorker, *, job_key: str | None = None, run_id: str | None = None +) -> None: + """Wait for the real long-running query before exercising the administrative cancel command.""" + active: dict[str, Any] = running_statement(worker, job_key=job_key) + cancelled: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "cancel-run", + run_id or active["run_id"], + "--event-id", + str(uuid4()), + "--reason", + "Cancel the active test statement.", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert cancelled.returncode == 0, cancelled.stderr + + +def wait_for_run( + api: SnapshotApi, + process: subprocess.Popen[bytes], + log: Path, + *, + pending: frozenset[str] = frozenset({"queued", "launched", "running", "unresponsive"}), + job_key: str | None = None, + run_id: str | None = None, +) -> dict[str, Any]: + """A terminal API projection must follow the scheduler's actual isolated warehouse execution.""" + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + deadline: float = time.monotonic() + 30 + current: dict[str, Any] = {"status": "queued"} + while current["status"] in pending: + assert process.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + with urlopen(request, timeout=2) as response: + runs: list[dict[str, Any]] = json.load(response) + by_job: dict[str | None, dict[str, Any]] = {run["job_key"]: run for run in runs} + by_job[None] = next(iter(runs), current) + by_id: dict[str | None, dict[str, Any]] = {run["run_id"]: run for run in runs} + by_id[None] = by_job.get(job_key, current) + current = by_id.get(run_id, current) + time.sleep(0.02) + return current + + +def claimed_worker(directory: Path, api: SnapshotApi, publication: dict[str, Any]) -> ClaimedWorker: + """Provision only the launch boundary; verification and lifecycle reporting use real HTTP.""" + run_id: str = str(uuid4()) + claim_id: str = str(uuid4()) + token_id: str = str(uuid4()) + bearer: str = f"{token_id}.{secrets.token_hex(32)}" + now: datetime = datetime.now(UTC) + selection: list[str] = ["model:orders"] + request: dict[str, Any] = { + "run_id": run_id, + "location_id": api.location_id, + "target": publication["target"], + "snapshot_id": publication["snapshot_id"], + "image_id": None, + "kind": "job", + "origin": "manual", + "job_key": None, + "parent_run_id": None, + "root_run_id": run_id, + "trigger_id": None, + "tick_id": None, + "rerun_of_run_id": None, + "requested_by": "test-admin", + "priority": 0, + "selection": selection, + "write_set": selection, + "settings": {"full_refresh": True}, + "concurrency_groups": [], + "overlap": "allow", + } + with sqlite3.connect(directory / ".sqlbuild/control.sqlite3") as connection: + _ = connection.execute( + "INSERT INTO sqb_runs (run_id,location_id,target,snapshot_id,kind,origin,root_run_id," + "requested_by,selection_json,request_json,write_set_digest,status,queued_at," + "node_counts_json,last_event_storage_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + run_id, + api.location_id, + publication["target"], + publication["snapshot_id"], + "job", + "manual", + run_id, + "test-admin", + json.dumps(selection), + json.dumps(request), + hashlib.sha256(json.dumps(selection).encode()).hexdigest(), + "queued", + now.isoformat(), + json.dumps({"pending": 1}), + 0, + ), + ) + _ = connection.execute( + "INSERT INTO sqb_run_nodes (run_id,node_key,attempt,status,writes) " + "VALUES (?,'model:orders',1,'pending',TRUE)", + (run_id,), + ) + _ = connection.execute( + "INSERT INTO sqb_write_claims (target,node_key,run_id,claimed_at) VALUES (?,?,?,?)", + (publication["target"], "model:orders", run_id, now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_run_liveness (run_id,worker_instance,claimed_at,last_heartbeat_at) " + "VALUES (?,?,?,?)", + (run_id, claim_id, now.isoformat(), now.isoformat()), + ) + _ = connection.execute( + "INSERT INTO sqb_access_tokens " + "(token_id,secret_hash,run_id,claim_id,scope_json,created_at,expires_at) " + "VALUES (?,?,?,?,?,?,?)", + ( + token_id, + hashlib.sha256(bearer.encode()).hexdigest(), + run_id, + claim_id, + json.dumps({"kind": "run", "run_id": run_id, "claim_id": claim_id}), + now.isoformat(), + (now + timedelta(hours=1)).isoformat(), + ), + ) + return ClaimedWorker(run_id=run_id, claim_id=claim_id, bearer=bearer) + + +def wait_for_sensor_phase( + store: Path, phase: str, *, empty_runs: bool = False, timeout: float = 45 +) -> dict[str, Any]: + """Wait for real scheduler facts, optionally requiring a completed skip.""" + deadline: float = time.monotonic() + timeout + row: tuple[str] | None = None + while row is None: + assert time.monotonic() < deadline, f"Sensor did not record {phase}" + with sqlite3.connect(store) as control: + row = control.execute( + "SELECT fact_json FROM sqb_sensor_evaluation_events WHERE phase=? " + "AND (? = FALSE OR fact_json->'outcome'->'run_ids'='[]') " + "ORDER BY storage_id DESC LIMIT 1", + (phase, empty_runs), + ).fetchone() + time.sleep(0.05) + return json.loads(row[0]) + + +def sensor_calls(path: Path, fields: dict[str, Any]) -> list[dict[str, Any]]: + """Select recorded callback invocations without inferring execution from declarations.""" + records: list[dict[str, Any]] = [json.loads(line) for line in path.read_text().splitlines()] + return list(filter(lambda record: fields.items() <= record.items(), records)) + + +def append_retried_fingerprints( + *, + adapter: Any, + connection: Any, + database_path: Path, + fingerprint: Fingerprint, + read_columns: Callable[..., Any] | None = None, +) -> None: + """Retry the same observation without substituting its identity or live catalog.""" + del database_path + for _attempt in range(2): + _ = write_fingerprint( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + fingerprint=fingerprint, + ensure_table=False, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + read_columns=read_columns or adapter.get_columns, + render_add_columns=adapter.render_add_columns, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + ) + + +def append_contended_fingerprints( + *, adapter: Any, connection: Any, database_path: Path, fingerprint: Fingerprint +) -> None: + """Release a real SQLite write lock only after the first ALTER has failed.""" + with closing(sqlite3.connect(database_path)) as blocker: + _ = adapter.execute(connection=connection, sql="PRAGMA busy_timeout=1") + catalog_actions: Iterator[Callable[[], Any]] = iter( + (lambda: blocker.execute("BEGIN IMMEDIATE"), blocker.commit) + ) + + def read_columns(**scope: Any) -> Any: + columns: Any = adapter.get_columns(**scope) + next(catalog_actions, lambda: None)() + return columns + + try: + append_retried_fingerprints( + adapter=adapter, + connection=connection, + database_path=database_path, + fingerprint=fingerprint, + read_columns=read_columns, + ) + finally: + blocker.rollback() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py new file mode 100644 index 0000000000..9d54256fc6 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py @@ -0,0 +1,164 @@ +"""Explicit backfills and their reruns execute the same pinned window through real workers.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + BackfillExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduled_worker, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + BackfillExecutionTestCase( + description="local integer window overrides authored bounds and survives group rerun", + model_sql="SELECT 1 AS amount", + warehouse_query="SELECT amount FROM orders ORDER BY amount", + expected_status="success", + expected_rows=[(23,), (37,)], + files={ + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/backfill_integer.sql" + ).read_text() + }, + window_arguments=("--start-cursor-int", "3", "--end-cursor-int", "6"), + expected_window={"kind": "integer", "start": 3, "end": 6}, + ), + BackfillExecutionTestCase( + description="Docker timestamp window normalizes offsets and survives group rerun", + model_sql="SELECT 1 AS amount", + warehouse_query="SELECT amount FROM orders ORDER BY amount", + expected_status="success", + expected_rows=[(23,), (37,)], + launcher="docker", + files={ + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/backfill_timestamp.sql" + ).read_text() + }, + window_arguments=( + "--start-cursor-ts", + "2026-07-03T05:30:00+05:30", + "--end-cursor-ts", + "2026-07-04T05:30:00+05:30", + ), + expected_window={ + "kind": "timestamp", + "start": "2026-07-03T00:00:00Z", + "end": "2026-07-04T00:00:00Z", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_only_its_frozen_window_is_written( + test_case: BackfillExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace( + test_case, + files=test_case.files + | { + "sources/raw_orders.yml": "sources:\n - name: raw_orders\n description: Synthetic order windows.\n schema: main\n table: raw_orders\n", + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "orders = job('orders', description='Windowed orders.', select='orders', " + "settings={'start_cursor': '0', 'end_cursor': '1'})\n" + "annual_orders = job_group('annual_orders', description='Windowed group.', jobs=[orders])\n" + ), + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + warehouse.execute( + "CREATE TABLE raw_orders AS SELECT * FROM (VALUES (11, 2, TIMESTAMP '2026-07-02'), (23, 3, TIMESTAMP '2026-07-03'), (37, 6, TIMESTAMP '2026-07-04'), (41, 7, TIMESTAMP '2026-07-05')) t(amount, batch_id, ordered_at)" + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + run_id: str = str(uuid4()) + arguments: list[str] = [ + executable, + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "request-run", + worker.api.location_id, + "annual_orders", + "--target", + "development", + "--run-id", + run_id, + "--reason", + "Backfill an explicit order window", + *test_case.window_arguments, + ] + rejected: subprocess.CompletedProcess[str] = subprocess.run( + arguments[:-2], capture_output=True, text=True, timeout=10, check=False + ) + assert rejected.returncode != 0 + assert "both start and end of exactly one cursor type" in rejected.stderr + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_runs").fetchone() == (0,) + for expected in [True, False]: + requested: subprocess.CompletedProcess[str] = subprocess.run( + arguments, capture_output=True, text=True, timeout=10, check=False + ) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"run_id": run_id, "applied": expected} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + warehouse.execute("UPDATE raw_orders SET amount=amount+100") + retry_id: str = str(uuid4()) + retried: subprocess.CompletedProcess[str] = subprocess.run( + [ + executable, + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "rerun-run", + run_id, + "--run-id", + retry_id, + "--reason", + "Repeat the same explicit order window", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert retried.returncode == 0, retried.stderr + retry: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=retry_id + ) + assert retry["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == [(123,), (137,)] + with sqlite3.connect(worker.control_store) as control: + requests: list[dict[str, Any]] = [ + json.loads(row[0]) + for row in control.execute("SELECT request_json FROM sqb_runs").fetchall() + ] + assert len(requests) == 4 + assert all(request["window"] == test_case.expected_window for request in requests) + assert {request["origin"] for request in requests} == {"backfill", "rerun"} + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM sqb_security_audit WHERE action='job.run_requested'" + ).fetchone() == (1,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py new file mode 100644 index 0000000000..52fabe8463 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -0,0 +1,292 @@ +"""Calendar callbacks remain in locations; native publication and refresh preserve good state.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.error import HTTPError + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + CalendarRefreshTestCase, + CalendarRejectionTestCase, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + scheduled_worker, + snapshot_api, + wait_for_generation, + wait_for_location_health, + wait_for_run, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + CalendarRejectionTestCase( + "naive time", "[datetime(2026, 1, 1)]", "timezone-aware datetimes" + ), + CalendarRejectionTestCase("exclusive end", "[window.end]", "half-open evaluation window"), + CalendarRejectionTestCase( + "before start", + "[window.start - timedelta(microseconds=1)]", + "half-open evaluation window", + ), + CalendarRejectionTestCase( + "non-datetime", "['2026-01-01T00:00:00Z']", "timezone-aware datetimes" + ), + CalendarRejectionTestCase( + "unbounded generator", "repeat(window.start)", "exceeds 10000 upcoming times" + ), + CalendarRejectionTestCase("zero horizon", "[]", "positive fixed duration", horizon="0s"), + CalendarRejectionTestCase( + "overflowing horizon", + "[]", + "positive fixed duration", + horizon="999999999999999999999999999d", + ), + CalendarRejectionTestCase("unknown zone", "[]", "IANA timezone", timezone="Unknown/Orders"), + ], + ids=lambda case: case.description, +) +def test_given_invalid_calendar_results_when_publishing_through_the_cli_then_validation_is_bounded_and_no_snapshot_is_committed( + test_case: CalendarRejectionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/orders.py": ( + "from datetime import datetime, timedelta\nfrom itertools import repeat\n" + "from sqlbuild.orchestration import calendar_schedule, job\n" + "orders = job('orders', description='Order reporting.', select='orders')\n" + f"@calendar_schedule(job=orders, description='Delivery times.', horizon={test_case.horizon!r}, timezone={test_case.timezone!r})\n" + f"def deliveries(window):\n return {test_case.expression}\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "publish", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + "--generation", + "1", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode != 0 + assert test_case.expected_message in result.stderr + with pytest.raises(HTTPError) as failure: + current_snapshot(api, "development") + assert failure.value.code == 404 + + +@pytest.mark.parametrize( + "test_case", + [ + CalendarRefreshTestCase("offset-aware calendar refresh", "America/New_York"), + CalendarRefreshTestCase("UTC calendar refresh", "UTC"), + ], + ids=lambda case: case.description, +) +def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refresh_reuses_the_interpreter_and_keeps_last_good( + test_case: CalendarRefreshTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + data: Path = tmp_path / "fulfillment.json" + evaluations: Path = tmp_path / "evaluations.jsonl" + data.write_text(json.dumps({"offsets": [3, 0, 3]})) + monkeypatch.setenv("SQLBUILD_FULFILLMENT_DATA", str(data)) + monkeypatch.setenv("SQLBUILD_CALENDAR_EVALUATIONS", str(evaluations)) + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py" + ) + .read_text() + .replace("America/New_York", test_case.expected_timezone) + + ( + "\n@calendar_schedule(job=orders, description='Other target.', horizon='8s', timezone='UTC', targets=('production',))\n" + "def excluded_calendar(window):\n raise ValueError('excluded calendar must not evaluate')\n" + ), + "orchestration/platform_orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ).read_text(), + "providers/fulfillment.py": ( + "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta, timezone\n" + "from sqlbuild.provider.classes.provider import Provider\n" + "class Fulfillment(Provider):\n" + ' """Upcoming synthetic fulfillment times."""\n' + " def setup(self, ctx):\n self.record({'phase': 'setup'})\n" + " def teardown(self):\n self.record({'phase': 'teardown'})\n" + " def record(self, value):\n" + " with Path(os.environ['SQLBUILD_CALENDAR_EVALUATIONS']).open('a') as out:\n" + " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" + " def upcoming(self, window):\n" + " self.record({'phase': 'evaluate', 'start': window.start.isoformat(), 'end': window.end.isoformat(), 'zone': window.start.tzinfo.key})\n" + " data = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())\n" + " if data.get('fail'):\n raise ValueError('synthetic calendar unavailable')\n" + " start = window.start.astimezone(timezone.utc)\n" + " return [start + timedelta(seconds=offset) for offset in data['offsets']]\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + log: Path = tmp_path / "location.log" + publications: Path = tmp_path / "publications.jsonl" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, publications) as process: + first: dict[str, Any] = wait_for_generation(api, "development", 1, process, log) + second: dict[str, Any] = wait_for_generation(api, "development", 2, process, log) + assert first["nodes"] == second["nodes"] + assert second["changes"] == [] + assert first["documents"]["triggers"].keys() == { + "fulfillment_calendar", + "overdue_orders", + } + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: + stored, next_due = control.execute( + "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id) WHERE kind = 'schedule'" + ).fetchone() + original_times: list[str] = first["documents"]["triggers"]["fulfillment_calendar"][ + "schedule" + ]["times"] + assert json.loads(stored)["schedule"]["times"] == sorted( + original_times + + second["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"] + ) + assert datetime.fromisoformat(next_due) == datetime.fromisoformat(original_times[0]) + observations: list[dict[str, Any]] = [ + json.loads(line) for line in evaluations.read_text().splitlines() + ] + assert [row["phase"] for row in observations] == ["setup", "evaluate", "teardown"] * 2 + successful: list[dict[str, Any]] = observations[1::3] + assert len(successful) == 2 + assert len({row["pid"] for row in observations}) == 1 + assert all(row["zone"] == test_case.expected_timezone for row in successful) + for snapshot, evaluation in zip([first, second], successful, strict=True): + start: datetime = datetime.fromisoformat(evaluation["start"]) + end: datetime = datetime.fromisoformat(evaluation["end"]) + assert end - start == timedelta(seconds=8) + schedule: dict[str, Any] = snapshot["documents"]["triggers"][ + "fulfillment_calendar" + ]["schedule"] + assert schedule["kind"] == "calendar" + assert [datetime.fromisoformat(value) for value in schedule["times"]] == [ + start + timedelta(seconds=offset) for offset in [0, 3] + ] + assert datetime.fromisoformat(schedule["refresh_at"]) == start + timedelta( + seconds=4 + ) + data.write_text(json.dumps({"fail": True})) + _ = wait_for_location_health( + api, "development", {"kind": "compile_failed", "generation": 3}, process, log + ) + assert current_snapshot(api, "development")["record"] == second["record"] + observations = [json.loads(line) for line in evaluations.read_text().splitlines()] + assert [row["phase"] for row in observations] == ["setup", "evaluate", "teardown"] * 3 + data.write_text(json.dumps({"offsets": [1]})) + (project / "models/stg_orders.sql").write_text( + TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(19 AS INTEGER)" + ) + ) + third: dict[str, Any] = wait_for_generation(api, "development", 3, process, log) + assert "CAST(19 AS INTEGER)" in third["nodes"]["model:stg_orders"]["raw_sql"] + assert ( + len(third["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"]) + == 1 + ) + assert [ + json.loads(line)["generation"] for line in publications.read_text().splitlines() + ] == [1, 2, 3] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="calendar evaluation belongs only to the location", + model_sql="SELECT 29 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(29,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_published_calendar_when_a_real_worker_compiles_then_it_does_not_evaluate_the_calendar_again( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + evaluated: Path = tmp_path / "evaluated" + test_case = replace( + test_case, + files=test_case.files + | { + "orchestration/orders.py": ( + "from pathlib import Path\nfrom sqlbuild.orchestration import calendar_schedule, job, schedule\n" + "orders = job('orders', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "@calendar_schedule(job=orders, description='Upcoming deliveries.', horizon='1d', timezone='UTC')\n" + "def deliveries(window):\n" + f" marker = Path({str(evaluated)!r})\n" + " if marker.exists():\n raise ValueError('calendar evaluated inside a run worker')\n" + " marker.write_text('evaluated')\n return []\n" + ) + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + assert evaluated.read_text() == "evaluated" + completed: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py new file mode 100644 index 0000000000..1c6fbcdc4c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -0,0 +1,160 @@ +"""A real location callback times out, restarts and queues a real DuckDB worker.""" + +import ast +import json +import os +import sqlite3 +import sys +from pathlib import Path +from typing import Any, cast + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + scheduled_worker, + sensor_calls, + wait_for_generation, + wait_for_run, + wait_for_sensor_phase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "polling fulfillment sensor", + "SELECT 83 AS amount", + "SELECT amount FROM orders", + "success", + [(83,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" + ).read_text(), + "providers/fulfillment.py": ( + "import json, os, time\nfrom pathlib import Path\n" + "from sqlbuild.orchestration import RunRequest, SkipReason\n" + "from sqlbuild.provider.classes.provider import Provider\n" + "class Fulfillment(Provider):\n" + ' """Synthetic fulfillment readiness."""\n' + " def record(self, value):\n" + " with Path(os.environ['SQLBUILD_SENSOR_CALLS']).open('a') as out:\n" + " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" + " def setup(self, ctx):\n self.record({'phase': 'setup'})\n" + " def teardown(self):\n self.record({'phase': 'teardown'})\n" + " def poll(self, ctx):\n" + " mode = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())['mode']\n" + " self.record({'phase': 'evaluate', 'cursor': ctx.cursor, 'evaluation_id': ctx.evaluation_id, 'target': ctx.target, 'mode': mode})\n" + " if mode == 'failed':\n" + " ctx.cursor = 'uncommitted'\n" + " raise ValueError('synthetic source unavailable')\n" + " if mode == 'timeout':\n time.sleep(90)\n" + " if ctx.cursor:\n return SkipReason('No new orders')\n" + " ctx.target = 'not-an-authority-input'\n" + " ctx.evaluation_id = 'not-an-authority-input'\n" + " ctx.cursor = 'orders/page-1'\n" + " return [RunRequest(run_key='orders-1'), RunRequest(run_key='orders-1')]\n" + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a_fresh_process_retries_without_advancing_its_cursor( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + context_module: ast.Module = ast.parse( + Path("src/sqlbuild/orchestration_declarations/models/sensor_context.py").read_text() + ) + assert [type(node).__name__ for node in context_module.body] == [ + "Expr", + "ImportFrom", + "ClassDef", + ] + declaration: ast.ClassDef = cast(ast.ClassDef, context_module.body[-1]) + assert declaration.name == "SensorContext" + assert [type(node).__name__ for node in declaration.body] == [ + "Expr", + "AnnAssign", + "AnnAssign", + "AnnAssign", + ] + data: Path = tmp_path / "fulfillment.json" + calls: Path = tmp_path / "sensor-calls.jsonl" + data.write_text(json.dumps({"mode": "failed"})) + monkeypatch.setenv("SQLBUILD_FULFILLMENT_DATA", str(data)) + monkeypatch.setenv("SQLBUILD_SENSOR_CALLS", str(calls)) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", worker.api.bearer) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(worker.project), + "location", + "serve", + "--server", + worker.api.url, + "--location-id", + worker.api.location_id, + "--target", + "development", + ], + log, + tmp_path / "publications.jsonl", + ) as location: + _ = wait_for_generation(worker.api, "development", 2, location, log) + failed: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "failed") + assert failed["request"]["cursor"] == {"event_storage_id": 0} + data.write_text(json.dumps({"mode": "timeout"})) + timed_out: dict[str, Any] = wait_for_sensor_phase(worker.control_store, "timed_out") + assert timed_out["request"]["cursor"] == failed["request"]["cursor"] + timed_call: dict[str, Any] = sensor_calls( + calls, {"evaluation_id": timed_out["request"]["evaluation_id"]} + )[0] + with pytest.raises(ProcessLookupError): + os.kill(timed_call["pid"], 0) + data.write_text(json.dumps({"mode": "ready"})) + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, log.read_text() + # The next successful evaluation skips but retains its committed cursor. + skipped: dict[str, Any] = wait_for_sensor_phase( + worker.control_store, "completed", empty_runs=True + ) + assert skipped["outcome"]["cursor"]["custom"] == "orders/page-1" + evaluated: list[dict[str, Any]] = sensor_calls(calls, {"phase": "evaluate"}) + assert {record["target"] for record in evaluated} == {"development"} + assert len({record["pid"] for record in evaluated}) == len(evaluated) + assert { + record["cursor"] + for record in sensor_calls(calls, {"phase": "evaluate", "mode": "failed"}) + } == {None} + assert { + record["cursor"] + for record in sensor_calls(calls, {"phase": "evaluate", "mode": "timeout"}) + } == {None} + assert location.poll() is None and worker.process.poll() is None + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT run_key,run_id FROM sqb_run_requests" + ).fetchall() == [("orders-1", destination["run_id"])] + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect( + str(worker.project / "warehouse.duckdb"), read_only=True + ) as warehouse: + assert ( + warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + ) + assert not (worker.project / ".sqlbuild/control.sqlite3").exists() + assert worker.api.bearer not in log.read_text() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py new file mode 100644 index 0000000000..05a6f35e07 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -0,0 +1,333 @@ +"""A published schedule executes in a real pinned container, not a mocked launcher.""" + +import signal +import sqlite3 +import sys +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from threading import Event +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication import helpers +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ScheduledWorker, + WorkerDeployment, + WorkerExecutionTestCase, + WorkerRestartTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import ( + REQUIRES_DOCKER, + docker_command, + docker_deployment, + lost_start_response, + worker_container, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + location_process, + running_statement, + scheduled_worker, + wait_for_log, + wait_for_run, + worker_result, +) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "pinned container inherits project context but not privileged credentials", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('37')) t(value)", + "SELECT amount FROM orders", + "success", + [(37,)], + launcher="docker", + files={ + "orchestration/orders.py": ( + "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" + " assert os.getenv('SQLBUILD_WORKER_TEST_VALUE') == 'orders-context'\n" + " for name in ('SQLBUILD_LOCATION_TOKEN', 'SQLBUILD_CONTROL_DATABASE_URL', " + "'SQLBUILD_SCHEDULER_TEST_STORE', 'SQLBUILD_PLATFORM_WEBHOOK_URL'):\n" + " assert name not in os.environ, name\n" + 'annual_orders = job("annual_orders", select="orders", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ) + }, + ), + WorkerExecutionTestCase( + "container warehouse error retains failed node evidence", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + launcher="docker", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_docker_job_when_published_then_a_pinned_worker_executes_and_reports( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert worker.docker is not None + deadline: float = time.monotonic() + 10 + state: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, run["run_id"] + ) + while state["Running"]: + assert time.monotonic() < deadline, "Terminal worker did not physically exit" + time.sleep(0.02) + state = worker_container(worker.docker, worker.api.location_id, run["run_id"]) + assert state["ExitCode"] == {"success": 0, "failed": 1}[test_case.expected_status] + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "container refuses changed source before writing", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="docker", + files_after_publication={ + "models/orders.sql": "MODEL (description 'Orders.', materialized table);\nSELECT 41 AS amount\n" + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_pinned_snapshot_when_container_source_drifts_then_no_warehouse_write_is_permitted( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert run["node_counts_json"] == {"skipped": 1} + assert not (worker.project / "warehouse.duckdb").exists() + assert worker.docker is not None + name: str = f"sqlbuild-{worker.api.location_id}-{run['run_id']}" + assert "node content differs" in docker_command(worker.docker.socket, "logs", name) + assert ( + worker_container(worker.docker, worker.api.location_id, run["run_id"])["ExitCode"] == 1 + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "Docker cancellation stops an active warehouse statement", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_cancelled_then_the_statement_stops_before_claim_release( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + cancel_running_statement(worker) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "killed Docker worker preserves its physical exit code", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + launcher="docker", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_killed_then_the_monitor_records_exit_and_releases_claims( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + assert worker.docker is not None + name: str = f"sqlbuild-{worker.api.location_id}-{active['run_id']}" + _ = docker_command(worker.docker.socket, "kill", "--signal", "KILL", name) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert ( + worker_container(worker.docker, worker.api.location_id, run["run_id"])["ExitCode"] + == 137 + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (137,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerRestartTestCase( + "graceful scheduler restart retains the executing container", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + shutdown_signal=signal.SIGTERM, + expected_scheduler_exit=0, + ), + WorkerRestartTestCase( + "abrupt scheduler death retains the executing container", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + launcher="docker", + shutdown_signal=signal.SIGKILL, + expected_scheduler_exit=-signal.SIGKILL, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_container_when_scheduler_restarts_then_the_same_worker_is_adopted_without_relaunch( + test_case: WorkerRestartTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + assert worker.docker is not None + before: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, active["run_id"] + ) + worker.process.send_signal(test_case.shutdown_signal) + assert worker.process.wait(timeout=10) == test_case.expected_scheduler_exit, ( + worker.log.read_text() + ) + assert worker_container(worker.docker, worker.api.location_id, active["run_id"])["Running"] + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + replacement: ScheduledWorker = worker._replace(process=process, log=log) + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + assert ( + worker_container(worker.docker, worker.api.location_id, active["run_id"])["Pid"] + == before["Pid"] + ) + cancel_running_statement(replacement) + run: dict[str, Any] = worker_result(replacement, test_case) + assert run["status"] == test_case.expected_status, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) + + +@REQUIRES_DOCKER +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "worker completes before the lost launch response can be durably observed", + "SELECT 37 AS amount", + "SELECT amount FROM orders", + "success", + [(37,)], + launcher="docker", + store_setup=( + "CREATE TRIGGER reject_launch_observation BEFORE INSERT ON recall_events " + "WHEN NEW.event_type='platform_launch_failed' " + "BEGIN SELECT RAISE(ABORT,'synthetic observation outage'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_lost_docker_start_response_when_the_worker_finishes_then_late_observation_retries_do_not_revive_it( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + inspection: Event = Event() + + @contextmanager + def deployment(directory: Path, executable: str) -> Iterator[WorkerDeployment]: + with docker_deployment(directory, executable) as actual: + assert actual.docker is not None + with lost_start_response(directory, actual.docker.socket, inspection) as proxy: + yield actual._replace(configuration=actual.configuration | {"socket": str(proxy)}) + + monkeypatch.setattr(helpers, "docker_deployment", deployment) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + wait_for_log(worker.process, worker.log, "Retrying worker shutdown publication") + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + before: list[tuple[Any, ...]] = control.execute("SELECT * FROM sqb_runs").fetchall() + assert control.execute( + "SELECT COUNT(*) FROM recall_events WHERE event_type IN ('run_launch_failed','platform_launch_failed')" + ).fetchone() == (0,) + _ = control.execute("DROP TRIGGER reject_launch_observation") + deadline: float = time.monotonic() + 10 + while control.execute( + "SELECT COUNT(*) FROM recall_events WHERE event_type='platform_launch_failed'" + ).fetchone() != (1,): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + # Terminal state alone cannot bypass the still-unavailable physical observation. + time.sleep(1.2) + assert worker.process.poll() is None, worker.log.read_text() + assert control.execute("SELECT * FROM sqb_runs").fetchall() == before + assert control.execute( + "SELECT event_type FROM recall_events WHERE event_type IN " + "('run_terminated','run_launch_failed','platform_launch_failed') ORDER BY storage_id" + ).fetchall() == [ + ("run_terminated",), + ("run_launch_failed",), + ("platform_launch_failed",), + ] + assert control.execute("SELECT COUNT(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT COUNT(*) FROM sqb_run_liveness").fetchone() == (0,) + inspection.set() + assert worker.process.wait(timeout=5) != 0, "Original response failure remains visible" + assert worker.docker is not None + state: dict[str, object] = worker_container( + worker.docker, worker.api.location_id, run["run_id"] + ) + assert state["Running"] is False + assert state["ExitCode"] == 0 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py new file mode 100644 index 0000000000..938a1b34b3 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -0,0 +1,184 @@ +"""A replacement Linux dev scheduler retains physical ownership of surviving local workers.""" + +import os +import signal +import sqlite3 +import sys +import time +from contextlib import suppress +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ScheduledWorker, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + location_process, + running_statement, + scheduled_worker, + wait_for_interpreter, + wait_for_log, + wait_for_run, + worker_result, +) + +fcntl: ModuleType = pytest.importorskip("fcntl") + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux inherited-lock and pidfd recovery") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "scheduler SIGKILL does not orphan a running local warehouse statement", + Path( + "benchmarks/duckshop/edge_cases/orchestration/local_worker_recovery.sql" + ).read_text(), + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + ) + ], + ids=lambda case: case.description, +) +def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_and_cancels_without_relaunch( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + active: dict[str, Any] = running_statement(worker) + with sqlite3.connect(worker.control_store) as control: + claim_id: str = control.execute( + "SELECT worker_instance FROM sqb_run_liveness WHERE run_id=?", (active["run_id"],) + ).fetchone()[0] + lock: Path = worker.project / ".sqlbuild/workers" / f"{claim_id}.lock" + with lock.open("rb") as ownership: + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + worker.process.kill() + assert worker.process.wait(timeout=10) == -signal.SIGKILL + with pytest.raises(BlockingIOError): + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + replacement: ScheduledWorker = worker._replace(process=process, log=log) + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + cancel_running_statement(replacement) + run: dict[str, Any] = worker_result(replacement, test_case) + assert run["status"] == test_case.expected_status, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_claimed'" + ).fetchone() == (1,) + deadline: float = time.monotonic() + 10 + while True: + try: + fcntl.flock(ownership, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.02) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux inherited-lock and pidfd recovery") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "pre-handshake child survives restart and failed terminal publication retains its claim", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="uncooperative", + store_setup=( + "CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status='failed' " + "BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_an_unstarted_orphan_when_recovery_publication_fails_then_claims_wait_for_physical_exit_and_commit( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + pid: int = wait_for_interpreter(worker.process, worker.project / "worker.pid", worker.log) + active: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, pending=frozenset({"queued"}) + ) + assert active["status"] == "launched" + handle: int = os.pidfd_open(pid) + try: + worker.process.kill() + assert worker.process.wait(timeout=10) == -signal.SIGKILL + log: Path = tmp_path / "replacement-scheduler.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "scheduler", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "--launchers", + str(tmp_path / "launchers.json"), + ], + log, + tmp_path / "replacement-scheduler.stdout", + ) as process: + wait_for_log(process, log, "Worker supervision recovered for run", timeout=45) + signal.pidfd_send_signal(handle, 0) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + assert control.execute("SELECT status FROM sqb_runs").fetchone() == ( + "launched", + ) + signal.pidfd_send_signal(handle, signal.SIGKILL) + wait_for_log(process, log, "Retrying worker shutdown publication") + assert process.poll() is None, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 1, + ) + _ = control.execute("DROP TRIGGER reject_terminal") + assert process.wait(timeout=10) != 0, log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT status FROM sqb_runs").fetchone() == ( + test_case.expected_status, + ) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == ( + 0, + ) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (1,) + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (None,) + assert not (worker.project / "warehouse.duckdb").exists() + finally: + with suppress(ProcessLookupError): + signal.pidfd_send_signal(handle, signal.SIGKILL) + os.close(handle) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py new file mode 100644 index 0000000000..9b12212a01 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -0,0 +1,303 @@ +"""Real compile, native transport, authenticated server and durable publication.""" + +import json +import socket +import sqlite3 +import subprocess +from pathlib import Path +from typing import Any +from urllib.parse import quote +from urllib.request import Request, urlopen + +import pytest +from _pytest.capture import CaptureResult + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, + run_cli, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + SnapshotPublicationTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + RejectedPublicationOutput, + current_snapshot, + snapshot_api, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotPublicationTestCase( + description="ordinary target", + target="development", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + SnapshotPublicationTestCase( + description="target requiring URL path encoding", + target="preview #1", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_retained_compiler_data_when_publishing_then_http_updates_atomically_and_rejections_keep_last_good( + test_case: SnapshotPublicationTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + files: dict[str, str] = TYPE_PROOF_FILES | { + "orchestration/edge_cases/definitions.py": Path( + "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" + ).read_text(), + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + } + write_files(project_dir=project_dir, files=files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics) + with snapshot_api(tmp_path / "control") as api: + metadata: dict[str, Any] = METADATA | { + "location_id": api.location_id, + "target": test_case.target, + } + first: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, json.dumps(metadata), project_dir + ) + record: dict[str, Any] = json.loads(first.publish(api.url, api.bearer)) + assert record["generation"] == 1 + assert record["location_id"] == api.location_id + initial: dict[str, Any] = current_snapshot(api, test_case.target) + assert frozenset(initial["nodes"]) == test_case.expected_nodes + assert initial["nodes"]["model:orders"]["definition"]["deps"] == ["model:stg_orders"] + assert initial["documents"]["triggers"]["edge_snapshot_orders"]["catch_up"] == { + "all": {"max_runs": 3} + } + edited: dict[str, str] = files | { + "models/stg_orders.sql": TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(9 AS INTEGER)" + ) + } + write_files(project_dir=project_dir, files=edited) + changed: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + next_snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( + changed, json.dumps(metadata | {"generation": 2}), project_dir + ) + updated: dict[str, Any] = json.loads(next_snapshot.publish(api.url, api.bearer)) + assert updated["generation"] == test_case.expected_generation + assert updated["snapshot_id"] != record["snapshot_id"] + with pytest.raises(RuntimeError, match="HTTP 409"): + _ = first.publish(api.url, api.bearer) + with pytest.raises(RuntimeError, match="HTTP 401"): + _ = next_snapshot.publish(api.url, "invalid-credential") + current: dict[str, Any] = current_snapshot(api, test_case.target) + assert current["record"] == updated + assert "CAST(9 AS INTEGER)" in current["nodes"]["model:stg_orders"]["raw_sql"] + request: Request = Request( + f"{api.url}/api/locations/{api.location_id}/targets/" + f"{quote(test_case.target, safe='')}/triggers", + headers={"Authorization": f"Bearer {api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + triggers: list[dict[str, Any]] = json.load(response) + by_name: dict[str, dict[str, Any]] = {trigger["name"]: trigger for trigger in triggers} + assert len(triggers) == 4 + assert set(by_name) == { + "edge_snapshot_orders", + "edge_platform_orders", + "edge_job_outcomes", + "edge_group_outcomes", + } + scheduled: dict[str, Any] = by_name["edge_snapshot_orders"] + assert scheduled["status"] == "paused" + assert scheduled["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" + assert scheduled["definition_json"]["default_status"] == "paused" + assert scheduled["definition_json"]["catch_up"] == {"all": {"max_runs": 3}} + assert by_name["edge_platform_orders"]["status"] == "paused" + assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "stopped" + assert by_name["edge_platform_orders"]["definition_json"]["when"] == { + "kind": "platform_event", + "kinds": ["tick_overdue", "location_broken", "launch_failed"], + } + assert by_name["edge_job_outcomes"]["definition_json"]["when"] == { + "kind": "run_event", + "job_key": "edge_snapshot_orders", + "kinds": ["warn", "partial"], + } + assert by_name["edge_group_outcomes"]["definition_json"]["when"] == { + "kind": "run_event", + "job_key": "edge_snapshot_group", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], + } + assert by_name["edge_job_outcomes"]["status"] == "paused" + assert by_name["edge_group_outcomes"]["status"] == "paused" + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotPublicationTestCase( + description="configured development target", + target="development", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + SnapshotPublicationTestCase( + description="configured target with encoded path characters", + target="preview #1", + expected_generation=2, + expected_nodes=frozenset({"model:orders", "model:stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolated_from_store_and_failed_edits_keep_last_good( + test_case: SnapshotPublicationTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + files: dict[str, str] = TYPE_PROOF_FILES | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job\n" + 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + ), + } + write_files(project_dir=project_dir, files=files) + _ = subprocess.run(["git", "init", "-q", str(project_dir)], check=True, capture_output=True) + _ = subprocess.run( + [ + "git", + "-C", + str(project_dir), + "-c", + "user.name=Orders Contributor", + "-c", + "user.email=orders@example.com", + "commit", + "--allow-empty", + "-qm", + "Initial orders", + ], + check=True, + capture_output=True, + ) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + arguments: list[str] = [ + "--no-color", + "--project-dir", + str(project_dir), + "location", + "publish", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + test_case.target, + "--image-id", + "sha256:synthetic-orders-image", + ] + assert run_cli([*arguments, "--generation", "1"]) == 0 + first_output: CaptureResult[str] = capsys.readouterr() + first: dict[str, Any] = json.loads(first_output.out) + assert first["generation"] == 1 + assert "Location snapshot published." in first_output.err + initial: dict[str, Any] = current_snapshot(api, test_case.target) + assert frozenset(initial["nodes"]) == test_case.expected_nodes + assert initial["documents"]["jobs"]["daily_orders"]["selection"] == ["model:orders"] + assert not (project_dir / ".sqlbuild/control.sqlite3").exists() + assert not (project_dir / "warehouse.duckdb").exists() + with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as connection: + metadata: tuple[str, int, str, str] | None = connection.execute( + "SELECT git_revision, git_dirty, git_author, image_id FROM sqb_snapshots WHERE snapshot_id=?", + (first["snapshot_id"],), + ).fetchone() + assert metadata is not None + assert len(metadata[0]) == 40 + assert metadata[1:] == (1, "Orders Contributor", "sha256:synthetic-orders-image") + + staging: Path = project_dir / "models/stg_orders.sql" + _ = staging.write_text( + files["models/stg_orders.sql"].replace("CAST(5 AS INTEGER)", "CAST(11 AS INTEGER)") + ) + assert run_cli([*arguments, "--generation", "2"]) == 0 + second_output: CaptureResult[str] = capsys.readouterr() + second: dict[str, Any] = json.loads(second_output.out) + assert second["generation"] == test_case.expected_generation + updated: dict[str, Any] = current_snapshot(api, test_case.target) + assert {change["subject_key"] for change in updated["changes"]} == {"model:stg_orders"} + assert "CAST(11 AS INTEGER)" in updated["nodes"]["model:stg_orders"]["raw_sql"] + + _ = staging.write_text("SELECT FROM\n") + assert run_cli([*arguments, "--generation", "3"]) != 0 + failed_output: CaptureResult[str] = capsys.readouterr() + assert not failed_output.out + assert "Location command failed." in failed_output.err + assert current_snapshot(api, test_case.target)["record"] == second + + _ = staging.write_text(files["models/stg_orders.sql"]) + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", "invalid-credential") + assert run_cli([*arguments, "--generation", "3"]) != 0 + rejected_output: CaptureResult[str] = capsys.readouterr() + assert not rejected_output.out + assert "HTTP 401" in rejected_output.err + assert api.bearer not in "".join( + ( + first_output.out, + first_output.err, + second_output.out, + second_output.err, + failed_output.err, + rejected_output.err, + ) + ) + assert current_snapshot(api, test_case.target)["record"] == second + + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + with socket.socket() as unavailable: + unavailable.bind(("127.0.0.1", 0)) + disconnected: list[str] = list(arguments) + disconnected[arguments.index("--server") + 1] = ( + f"http://127.0.0.1:{unavailable.getsockname()[1]}" + ) + assert run_cli([*disconnected, "--generation", "3"]) != 0 + transport_output: CaptureResult[str] = capsys.readouterr() + assert not transport_output.out + assert "generation 3 may already be published" in transport_output.err + assert current_snapshot(api, test_case.target)["record"] == second + + with monkeypatch.context() as result_scope: + result_scope.setattr("sys.stdout", RejectedPublicationOutput()) + assert run_cli([*arguments, "--generation", "3"]) != 0 + delivery_output: CaptureResult[str] = capsys.readouterr() + assert "was published, but result delivery failed" in delivery_output.err + assert current_snapshot(api, test_case.target)["record"]["generation"] == 3 diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py new file mode 100644 index 0000000000..b8e7303b67 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -0,0 +1,356 @@ +"""A native supervisor reloads real project interpreters through the authenticated HTTP API.""" + +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationDatabaseTestCase, + LocationReloadTestCase, + LocationShutdownTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + current_snapshot, + location_process, + snapshot_api, + wait_for_generation, + wait_for_interpreter, + wait_for_location_health, + wait_for_log, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + LocationReloadTestCase( + "healthy connection", + "development", + 7, + int(sys.platform == "win32"), + "warehouse.duckdb", + True, + ), + LocationReloadTestCase( + "failed connection with encoded target", + "preview #1", + 7, + int(sys.platform == "win32"), + "missing/warehouse.duckdb", + False, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_served_location_when_inputs_change_or_restart_then_generations_reload_without_losing_last_good( + test_case: LocationReloadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + compiler_pid: Path = tmp_path / "compiler.pid" + declaration: str = ( + "from sqlbuild.orchestration import job\n" + "import os\nfrom pathlib import Path\n" + f"Path({str(compiler_pid)!r}).write_text(str(os.getpid()))\n" + 'print("Orders declaration loaded.")\n' + 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + ) + files: dict[str, str] = TYPE_PROOF_FILES | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', + "orchestration/orders.py": declaration, + "sinks/orders.py": 'print("Loading orders sink", flush=True)\n', + } + write_files(project_dir=project, files=files) + _ = subprocess.run(["git", "init", "-q", str(project)], check=True, capture_output=True) + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + # A presentation-only width fallback must not bypass the native-only supervisor. + # Keep the adversarial width out of pytest's own terminal reporter. + environment: dict[str, str] = os.environ | {"COLUMNS": "999999999999999999999999999999"} + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + test_case.target, + ] + log: Path = project / "location.log" + publications: Path = project / "publications.jsonl" + with location_process(arguments, log, publications, env=environment) as process: + first: dict[str, Any] = wait_for_generation(api, test_case.target, 1, process, log) + first_interpreter: int = int(compiler_pid.read_text()) + heartbeat: dict[str, Any] = wait_for_location_health( + api, test_case.target, {"kind": "heartbeat"}, process, log + ) + initial_health: dict[str, Any] = wait_for_location_health( + api, + test_case.target, + {"kind": "connection", "generation": 1, "healthy": test_case.expected_connection}, + process, + log, + ) + assert initial_health["report"]["session_id"] == heartbeat["report"]["session_id"] + staging: Path = project / "models/stg_orders.sql" + for value in (11, 12, 13): + _ = staging.write_text( + files["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", f"CAST({value} AS INTEGER)" + ) + ) + second: dict[str, Any] = wait_for_generation(api, test_case.target, 2, process, log) + assert second["record"]["snapshot_id"] != first["record"]["snapshot_id"] + assert int(compiler_pid.read_text()) == first_interpreter + assert "CAST(13 AS INTEGER)" in second["nodes"]["model:stg_orders"]["raw_sql"] + assert {change["subject_key"] for change in second["changes"]} == {"model:stg_orders"} + + _ = staging.write_text("SELECT FROM invalid SQL !\n") + _ = wait_for_location_health( + api, test_case.target, {"kind": "compile_failed", "generation": 3}, process, log + ) + assert current_snapshot(api, test_case.target)["record"] == second["record"] + _ = staging.write_text( + files["models/stg_orders.sql"].replace("CAST(5 AS INTEGER)", "CAST(14 AS INTEGER)") + ) + recovered: dict[str, Any] = wait_for_generation(api, test_case.target, 3, process, log) + assert int(compiler_pid.read_text()) == first_interpreter + assert "CAST(14 AS INTEGER)" in recovered["nodes"]["model:stg_orders"]["raw_sql"] + + _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") + wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") + _ = wait_for_location_health( + api, test_case.target, {"kind": "compile_failed", "generation": 4}, process, log + ) + assert current_snapshot(api, test_case.target)["record"] == recovered["record"] + _ = (project / "orchestration/orders.py").write_text( + declaration.replace("Build orders.", "Refresh order totals.") + ) + third: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) + replacement_interpreter: int = int(compiler_pid.read_text()) + assert replacement_interpreter != first_interpreter + assert ( + third["documents"]["jobs"]["daily_orders"]["description"] == "Refresh order totals." + ) + _ = subprocess.run( + [ + "git", + "-C", + str(project), + "-c", + "user.name=Orders Contributor", + "-c", + "user.email=orders@example.com", + "commit", + "--allow-empty", + "-qm", + "Record orders", + ], + check=True, + capture_output=True, + ) + committed: dict[str, Any] = wait_for_generation(api, test_case.target, 5, process, log) + assert int(compiler_pid.read_text()) == replacement_interpreter + assert committed["nodes"] == third["nodes"] + _ = (project / "sqlbuild_project.toml").write_text( + files["sqlbuild_project.toml"].replace('schema = "main"', 'schema = "reporting"') + ) + configured: dict[str, Any] = wait_for_generation(api, test_case.target, 6, process, log) + assert int(compiler_pid.read_text()) == replacement_interpreter + assert ( + configured["nodes"]["model:orders"]["definition"]["destination"]["schema"] + == "reporting" + ) + assert ( + configured["nodes"]["model:orders"]["raw_sql"] + == committed["nodes"]["model:orders"]["raw_sql"] + ) + process.terminate() + assert process.wait(timeout=5) == test_case.expected_stop_code + with location_process(arguments, log, publications, env=environment) as process: + restarted: dict[str, Any] = wait_for_generation( + api, test_case.target, test_case.expected_generation, process, log + ) + assert int(compiler_pid.read_text()) != replacement_interpreter + assert restarted["nodes"] == configured["nodes"] + restarted_health: dict[str, Any] = wait_for_location_health( + api, + test_case.target, + {"kind": "connection", "generation": 7, "healthy": test_case.expected_connection}, + process, + log, + ) + assert ( + restarted_health["report"]["session_id"] != initial_health["report"]["session_id"] + ) + assert restarted_health["received_at"] >= initial_health["received_at"] + # Redirected output and compiler cache writes must not feed the input watcher. + time.sleep(1) + assert current_snapshot(api, test_case.target)["record"] == restarted["record"] + process.terminate() + assert process.wait(timeout=5) == test_case.expected_stop_code + records: list[dict[str, Any]] = [ + json.loads(line) for line in publications.read_text().splitlines() + ] + assert [record["generation"] for record in records] == [1, 2, 3, 4, 5, 6, 7] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() + assert (project / "warehouse.duckdb").exists() == test_case.expected_connection + + +@pytest.mark.skipif(sys.platform == "win32", reason="Unix signal and process ownership contract") +@pytest.mark.parametrize( + "test_case", + [ + LocationShutdownTestCase( + "terminate compiling interpreter after repeated heartbeats", + signal.SIGTERM, + 0, + 2, + "runtime", + ), + LocationShutdownTestCase("interrupt compiling interpreter", signal.SIGINT, 0, 1, "runtime"), + LocationShutdownTestCase( + "terminate connection checking interpreter", signal.SIGTERM, 0, 1, "debug" + ), + ], + ids=lambda case: case.description, +) +def test_given_a_busy_location_when_stopped_then_its_interpreter_is_reaped( + test_case: LocationShutdownTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + child_pid: Path = tmp_path / "interpreter.pid" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + "adapters/slow.py": ( + "import os, sys, time\nfrom pathlib import Path\n" + f"if {test_case.blocked_command!r} in sys.argv:\n" + f" Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + " time.sleep(60)\n" + ), + }, + ) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + log: Path = tmp_path / "location.log" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + interpreter: int = wait_for_interpreter(process, child_pid, log) + _ = wait_for_location_health( + api, + "development", + {"kind": "heartbeat"}, + process, + log, + minimum_reports=test_case.minimum_heartbeats, + ) + process.send_signal(test_case.signal) + assert process.wait(timeout=5) == test_case.expected_exit_code + with pytest.raises(ProcessLookupError): + os.kill(interpreter, 0) + + +@pytest.mark.parametrize( + "test_case", + [ + LocationDatabaseTestCase( + "custom database extension", "warehouse.db", 1, 'database = "warehouse.db"' + ), + LocationDatabaseTestCase( + "extensionless database", "warehouse", 1, 'database = "warehouse"' + ), + LocationDatabaseTestCase( + "custom attached database", + "inventory.db", + 1, + 'database = ":memory:"\nattach = [{path = "inventory.db", alias = "inventory"}]', + ), + ], + ids=lambda case: case.description, +) +def test_given_a_location_using_a_local_database_when_it_is_created_and_written_then_only_source_edits_reload( + test_case: LocationDatabaseTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\n{test_case.connection_toml}\n" + + '\n[targets.development]\nschema = "main"\n', + }, + ) + database: Path = project / test_case.database + assert not database.exists() + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + log: Path = tmp_path / "location.log" + arguments: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ] + with location_process(arguments, log, tmp_path / "publications.jsonl") as process: + baseline: dict[str, Any] = wait_for_generation( + api, "development", test_case.expected_generation, process, log + ) + _ = wait_for_location_health( + api, + "development", + {"kind": "connection", "generation": 1, "healthy": True}, + process, + log, + ) + assert database.is_file() + with duckdb.connect(str(database)) as connection: + _ = connection.execute( + "CREATE TABLE orders (order_id INTEGER); INSERT INTO orders VALUES (17), (23)" + ) + assert connection.execute("SELECT sum(order_id) FROM orders").fetchone() == (40,) + time.sleep(1) + assert current_snapshot(api, "development")["record"] == baseline["record"] + time.sleep(1) + assert current_snapshot(api, "development")["record"] == baseline["record"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py new file mode 100644 index 0000000000..f7edf07364 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py @@ -0,0 +1,124 @@ +"""Pinned group retries rebuild failed and skipped nodes without repeating successful writes.""" + +import json +import sqlite3 +import subprocess +import sys +from dataclasses import replace +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduled_worker, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="local group retries only failed and skipped nodes", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM totals ORDER BY amount", + expected_status="success", + expected_rows=[(17,), (99,)], + ), + WorkerExecutionTestCase( + description="Docker group retries only failed and skipped nodes", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM totals ORDER BY amount", + expected_status="success", + expected_rows=[(17,), (99,)], + launcher="docker", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_are_not_rewritten( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_case = replace( + test_case, + files={ + "models/broken_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/rerun_from_failure.sql" + ).read_text(), + "models/totals.sql": ( + "MODEL (description 'Downstream totals.', materialized table);\n" + 'SELECT amount FROM __ref("broken_orders")\n' + ), + "models/inventory.sql": ( + "MODEL (description 'Independent inventory.', materialized table);\n" + "SELECT 29 AS amount\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "staging = job('staging', description='Stage orders.', select='orders broken_orders')\n" + "marts = job('marts', description='Build totals.', select='totals')\n" + "inventory = job('inventory', description='Build inventory.', select='inventory')\n" + "annual_orders = job_group('annual_orders', description='Retry boundaries.', " + "jobs=[inventory, marts, staging], schedule=schedule(cron='0 0 1 1 *', " + "timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + source: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert source["status"] == "partial", worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + warehouse.execute("CREATE TABLE retry_ready AS SELECT 1 AS id") + warehouse.execute("UPDATE orders SET amount=99") + warehouse.execute("UPDATE inventory SET amount=71") + run_id: str = str(uuid4()) + command: list[str] = [ + str(Path(sys.executable).with_name("sqb")), + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "rerun-run", + source["run_id"], + "--run-id", + run_id, + "--from-failure", + "--reason", + "Retry failed and skipped nodes after fixing warehouse input.", + ] + for expected_applied in [True, False]: + requested: subprocess.CompletedProcess[str] = subprocess.run( + command, capture_output=True, text=True, timeout=10, check=False + ) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"run_id": run_id, "applied": expected_applied} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + assert completed["snapshot_id"] == source["snapshot_id"] + assert completed["selection_json"] == ["model:broken_orders", "model:totals"] + assert completed["node_counts_json"] == {"success": 2} + with sqlite3.connect(worker.control_store) as control: + children: list[tuple[str, str]] = control.execute( + "SELECT job_key, selection_json FROM sqb_runs WHERE parent_run_id=? ORDER BY job_key", + (run_id,), + ).fetchall() + assert [(name, json.loads(selection)) for name, selection in children] == [ + ("marts", ["model:totals"]), + ("staging", ["model:broken_orders"]), + ] + assert control.execute( + "SELECT count(*) FROM sqb_security_audit WHERE action='run.rerun_requested'" + ).fetchone() == (1,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + assert warehouse.sql("SELECT amount FROM inventory").fetchall() == [(71,)] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py new file mode 100644 index 0000000000..2b328d4c5f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -0,0 +1,435 @@ +"""Real warehouse writers, populated predecessors and the isolated read-only history protocol.""" + +import hashlib +import json +import sqlite3 +from contextlib import closing +from datetime import UTC, datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + Fingerprint, + SourceFreshnessRecord, + SourceFreshnessRenderers, + read_latest_fingerprints, + read_latest_source_freshness, + write_fingerprints, + write_source_freshness_records, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + FingerprintHistoryTestCase, + WarehouseHistoryTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + append_contended_fingerprints, + append_retried_fingerprints, + open_legacy_warehouse, + warehouse_history_page, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WarehouseHistoryTestCase( + "DuckDB typed timestamps", + "duckdb", + (100, 6), + "database", + (), + append_retried_fingerprints, + ), + WarehouseHistoryTestCase( + "SQLite timestamp text", + "sqlite", + (100, 6), + "path", + ("delete",), + append_contended_fingerprints, + ), + ], + ids=lambda case: case.description, +) +def test_given_populated_predecessor_tables_when_writers_add_ids_then_history_preserves_new_legacy_and_retried_observations( + test_case: WarehouseHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + connection_key: str = test_case.connection_key + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + adapter.close(connection) + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) + + def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: + return warehouse_history_page(project, "source_freshness", after, observed_since) + + old: dict[str, Any] = page() + assert len(old["records"]) == 1 and old["records"][0]["event_id"] is None + assert page(observed_since="2026-01-01T09:00:00Z") == old + assert page(observed_since="2026-01-01T10:00:00.000001Z") == { + "records": [], + "next_after": None, + } + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) + connection = adapter.connect({connection_key: str(project / "warehouse.db")}) + # Reading history never upgrades the warehouse or imports project declarations. + assert "event_id" not in { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name="_sqlbuild_source_freshness" + ) + } + latest: Any = read_latest_fingerprints( + connection=connection, + execute=adapter.execute, + table_exists=True, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_read_latest_sql=adapter.render_read_latest_fingerprints_sql, + ) + assert latest.fingerprints["orders"].run_id == "legacy-run" + fingerprint: Fingerprint = Fingerprint( + node_type="model", + node_name="orders", + target_database=None, + target_schema="main", + target_name="orders", + run_id="external-run", + definition_hash="definition", + version_hash="version", + schema_fingerprint="schema", + definition="SELECT 2 AS order_id", + ts=datetime(2026, 2, 1, tzinfo=UTC), + ) + test_case.append_fingerprints( + adapter=adapter, + connection=connection, + database_path=project / "warehouse.db", + fingerprint=fingerprint, + ) + expected_fingerprint_id: str = hashlib.sha256( + json.dumps( + [ + "sqlbuild.fingerprint.event.v1", + "model", + "orders", + None, + "main", + "orders", + "external-run", + "version", + ], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + assert adapter.query( + connection=connection, + sql="SELECT event_id FROM main._sqlbuild_fingerprints ORDER BY ts", + limit=10, + ).rows == ( + (None,), + (expected_fingerprint_id,), + (expected_fingerprint_id,), + ) + records: tuple[SourceFreshnessRecord, ...] = tuple( + SourceFreshnessRecord( + source_name=("Orders", "orders", "orders ", "ordérs")[index % 4], + target_database=("", None)[index % 2], + target_schema="main", + target_name="orders", + run_id=f"external-{index}", + strategy="max_column", + value_kind="string", + data_version=("o'rders 🧾", None, None)[index % 3], + data_version_hash=f"value-{index}", + observed_at=datetime(2026, 2, 1, microsecond=index, tzinfo=UTC), + ) + for index in range(104) + ) + renderers: SourceFreshnessRenderers = SourceFreshnessRenderers( + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_insert_records_sql=adapter.render_insert_source_freshness_records_sql, + render_create_table_sql=adapter.render_create_source_freshness_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + for batch in (records, records[:1]): + write_source_freshness_records( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + records=batch, + renderers=renderers, + ) + expected_freshness_id: str = hashlib.sha256( + json.dumps( + [ + "sqlbuild.source-freshness.event.v1", + "Orders", + "", + "main", + "orders", + "external-0", + "2026-02-01T00:00:00.000000+00:00", + "max_column", + "string", + "value-0", + ], + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + assert adapter.query( + connection=connection, + sql="SELECT event_id FROM main._sqlbuild_source_freshness WHERE run_id='external-0'", + limit=10, + ).rows == ( + (expected_freshness_id,), + (expected_freshness_id,), + ) + # An equivalent offset denotes the same observation, not another event. + equivalent: SourceFreshnessRecord = SourceFreshnessRecord( + source_name="Orders", + target_database="", + target_schema="main", + target_name="orders", + run_id="external-0", + strategy="max_column", + value_kind="string", + data_version="o'rders 🧾", + data_version_hash="value-0", + observed_at=datetime(2026, 2, 1, 1, tzinfo=timezone(timedelta(hours=1))), + ) + assert expected_freshness_id in adapter.render_insert_source_freshness_records_sql( + database=None, schema="main", records=(equivalent,) + ) + current: Any = read_latest_source_freshness( + connection=connection, + execute=adapter.execute, + table_exists=True, + database=None, + schema="main", + render_qualified_name=adapter.render_qualified_name, + render_read_latest_sql=adapter.render_read_latest_source_freshness_sql, + ) + assert {row.run_id for row in current.records.values()} == { + "legacy-run", + "external-100", + "external-101", + "external-102", + "external-103", + } + adapter.close(connection) + pages: list[dict[str, Any]] = [page()] + while pages[-1]["next_after"] is not None: + pages.append(page(pages[-1]["next_after"])) + assert len(pages) < 4 + assert tuple(len(item["records"]) for item in pages) == test_case.expected_page_lengths + imported: list[dict[str, Any]] = [] + for item in pages: + imported.extend(item["records"]) + event_ids: set[str | None] = {row["event_id"] for row in imported} + assert len(event_ids - {None}) == 104 + assert sum(row["event_id"] is None for row in imported) == 1 + assert {row["run_id"] for row in imported} == {"legacy-run"} | { + f"external-{index}" for index in range(104) + } + assert {row["target_database"] for row in imported} == {None, ""} + # The time bound is inclusive and independent of source-name/keyset ordering. + # SQL may read a superset within the boundary day, so a short decoded page + # must retain its continuation if the raw query filled the page. + observed_since: str = "2026-02-01T01:00:00.000050+01:00" + recent: list[dict[str, Any]] = [page(observed_since=observed_since)] + while recent[-1]["next_after"] is not None: + recent.append(page(recent[-1]["next_after"], observed_since)) + assert len(recent) < 4 + recent_records: list[dict[str, Any]] = [] + for item in recent: + recent_records.extend(item["records"]) + assert len(recent_records) == 54 + assert {row["run_id"] for row in recent_records} == { + f"external-{index}" for index in range(50, 104) + } + zero: dict[str, Any] = page(observed_since="2026-02-01T00:00:00Z") + assert ( + len(zero["records"]) + len(page(zero["next_after"], "2026-02-01T00:00:00Z")["records"]) + == 105 + ) + assert sum(row["run_id"] == "external-0" for row in zero["records"]) == 2 + # An empty decoded page can still have another bounded SQL page to inspect. + empty_since: str = "2026-02-01T00:00:00.000104Z" + empty: dict[str, Any] = page(observed_since=empty_since) + assert empty["records"] == [] and empty["next_after"] is not None + assert page(empty["next_after"], empty_since) == {"records": [], "next_after": None} + assert page(observed_since="2030-01-01T00:00:00Z") == { + "records": [], + "next_after": None, + } + assert not (project / ".sqlbuild/control.sqlite3").exists() + (project / "warehouse.db").rename(project / "retained-warehouse.db") + assert page() is None + assert not (project / "warehouse.db").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + FingerprintHistoryTestCase( + "DuckDB typed fingerprint timestamps", + "duckdb", + "database", + datetime(2026, 2, 1, tzinfo=UTC), + (), + (), + ("legacy-run",), + ), + FingerprintHistoryTestCase( + "SQLite offset fingerprint timestamps", + "sqlite", + "path", + datetime(2026, 1, 31, 23, tzinfo=timezone(timedelta(hours=-1))), + ("delete",), + ( + Path( + "benchmarks/duckshop/edge_cases/orchestration/legacy_warehouse_fingerprint_offsets.sql" + ), + ), + ("legacy-run", "offset-run"), + ), + ], + ids=lambda case: case.description, +) +def test_given_fingerprint_history_when_read_in_isolated_pages_then_legacy_retries_definitions_and_time_bounds_survive( + test_case: FingerprintHistoryTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse( + project, test_case.adapter_name, test_case.connection_key + ) + for fixture in test_case.fixture_paths: + _ = adapter.execute(connection=connection, sql=fixture.read_text()) + adapter.close(connection) + + def page(after: list[str | None] | None = None, observed_since: str | None = None) -> Any: + return warehouse_history_page(project, "fingerprints", after, observed_since) + + legacy: dict[str, Any] = page() + assert legacy["next_after"] is None + assert {row["run_id"] for row in legacy["records"]} == set(test_case.expected_legacy_runs) + assert legacy["records"][0]["event_id"] is None + assert legacy["records"][0]["definition"] == "SELECT 1 AS order_id" + assert legacy["records"][0]["metadata_json"] == "{}" + assert page(observed_since="2026-01-01T09:00:00Z") == legacy + assert page(observed_since="2026-01-01T10:00:00.000001Z") == { + "records": [], + "next_after": None, + } + connection = adapter.connect({test_case.connection_key: str(project / "warehouse.db")}) + assert "event_id" not in { + column.name + for column in adapter.get_columns( + connection=connection, database=None, schema="main", name="_sqlbuild_fingerprints" + ) + } + fingerprints: tuple[Fingerprint, ...] = tuple( + Fingerprint( + node_type=("model", "seed")[index % 2], + node_name=("Orders", "orders", "orders ", "ordérs", "o'rders 🧾")[index % 5], + target_database=(None, "")[index % 2], + target_schema="main", + target_name="orders", + run_id=f"external-{index}", + definition_hash="definition", + version_hash=f"version-{index}", + schema_fingerprint="schema", + definition="SELECT 'o''rders 🧾' AS order_note", + metadata_json='{"note":"orders 🧾"}', + ts=test_case.observed_at + timedelta(microseconds=index), + ) + for index in range(104) + ) + for batch in (fingerprints, fingerprints[:1]): + write_fingerprints( + connection=connection, + execute=adapter.execute, + database=None, + schema="main", + fingerprints=batch, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + ) + adapter.close(connection) + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode=DELETE").fetchone() == (journal_mode,) + pages: list[dict[str, Any]] = [page()] + while pages[-1]["next_after"] is not None: + assert len(pages[-1]["next_after"]) == 13 + pages.append(page(pages[-1]["next_after"])) + assert len(pages) < 4 + assert tuple(len(item["records"]) for item in pages) == ( + 100, + 5 + len(test_case.expected_legacy_runs), + ) + imported: list[dict[str, Any]] = [] + for item in pages: + imported.extend(item["records"]) + assert {row["run_id"] for row in imported} == set(test_case.expected_legacy_runs) | { + f"external-{index}" for index in range(104) + } + assert len({row["event_id"] for row in imported} - {None}) == 104 + assert sum(row["event_id"] is None for row in imported) == len(test_case.expected_legacy_runs) + assert {row["target_database"] for row in imported} == {None, ""} + first: list[dict[str, Any]] = list(filter(lambda row: row["run_id"] == "external-0", imported)) + assert len(first) == 2 and first[0] == first[1] + assert first[0]["definition"] == "SELECT 'o''rders 🧾' AS order_note" + assert first[0]["metadata_json"] == '{"note":"orders 🧾"}' + assert ( + first[0]["event_id"] + == hashlib.sha256( + b'["sqlbuild.fingerprint.event.v1","model","Orders",null,"main","orders","external-0","version-0"]' + ).hexdigest() + ) + assert datetime.fromisoformat(first[0]["ts"]) == datetime(2026, 2, 1, tzinfo=UTC) + observed_since: str = "2026-02-01T01:00:00.000050+01:00" + recent: list[dict[str, Any]] = [page(observed_since=observed_since)] + while recent[-1]["next_after"] is not None: + recent.append(page(recent[-1]["next_after"], observed_since)) + assert len(recent) < 4 + recent_records: list[dict[str, Any]] = [] + for item in recent: + recent_records.extend(item["records"]) + assert len(recent_records) == 54 + assert {row["run_id"] for row in recent_records} == { + f"external-{index}" for index in range(50, 104) + } + empty_since: str = "2026-02-01T00:00:00.000104Z" + empty: dict[str, Any] = page(observed_since=empty_since) + assert empty["records"] == [] and empty["next_after"] is not None + assert page(empty["next_after"], empty_since) == {"records": [], "next_after": None} + for journal_mode in test_case.journal_modes: + with closing(sqlite3.connect(project / "warehouse.db")) as predecessor: + assert predecessor.execute("PRAGMA journal_mode").fetchone() == (journal_mode,) + assert not (project / ".sqlbuild/control.sqlite3").exists() + (project / "warehouse.db").rename(project / "retained-warehouse.db") + assert page() is None + assert not (project / "warehouse.db").exists() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py new file mode 100644 index 0000000000..502613c572 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_work.py @@ -0,0 +1,158 @@ +"""Trusted owner requests cross the real HTTP channel to an isolated DuckDB connection check.""" + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + LocationWorkTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + location_process, + snapshot_api, + wait_for_generation, + wait_for_interpreter, + wait_for_location_health, + wait_for_location_work, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + LocationWorkTestCase("healthy requested check", "warehouse.db", True), + LocationWorkTestCase("failed requested check", "missing/warehouse.db", False), + ], + ids=lambda case: case.description, +) +def test_given_a_published_location_when_an_owner_requests_work_then_results_return_outbound_and_stale_work_is_not_executed( + test_case: LocationWorkTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + blocked: Path = tmp_path / "block-debug" + child_pid: Path = tmp_path / "debug.pid" + write_files( + project_dir=project, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + + '\n[targets."preview #1"]\nschema = "main"\n', + "adapters/slow.py": ( + "import os, sys, time\nfrom pathlib import Path\n" + f"if 'debug' in sys.argv and Path({str(blocked)!r}).exists():\n" + f" Path({str(child_pid)!r}).write_text(str(os.getpid()))\n" + f" while Path({str(blocked)!r}).exists():\n" + " time.sleep(0.02)\n" + ), + }, + ) + executable: str = str(Path(sys.executable).with_name("sqb")) + with snapshot_api(tmp_path / "control") as api: + monkeypatch.setenv("SQLBUILD_LOCATION_TOKEN", api.bearer) + # The serving location must never open the owner's database. + monkeypatch.setenv("SQLBUILD_CONTROL_DATABASE_URL", "unsupported://must-not-open") + owner_environment: dict[str, str] = os.environ | { + "SQLBUILD_CONTROL_DATABASE_URL": str(tmp_path / "control/.sqlbuild/control.sqlite3") + } + arguments: list[str] = [ + executable, + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "preview #1", + ] + request_id: str = str(uuid4()) + enqueue: list[str] = [ + executable, + "admin", + "check-location", + api.location_id, + "--target", + "preview #1", + "--request-id", + request_id, + "--reason", + "Check orders connectivity", + ] + log: Path = tmp_path / "location.log" + publications: Path = tmp_path / "publications.jsonl" + with location_process(arguments, log, publications) as process: + _ = wait_for_generation(api, "preview #1", 1, process, log) + _ = wait_for_location_health( + api, + "preview #1", + {"kind": "connection", "generation": 1, "healthy": test_case.expected_connection}, + process, + log, + ) + first: subprocess.CompletedProcess[str] = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + assert json.loads(first.stdout) == {"request_id": request_id, "applied": True} + retry: subprocess.CompletedProcess[str] = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + assert json.loads(retry.stdout) == {"request_id": request_id, "applied": False} + result: dict[str, Any] = wait_for_location_work(api, request_id, process, log) + assert result["generation"] == 1 + assert result["result_json"]["outcome"] == { + "kind": "connection", + "healthy": test_case.expected_connection, + } + read: subprocess.CompletedProcess[str] = subprocess.run( + [executable, "admin", "location-work", request_id], + env=owner_environment, + capture_output=True, + text=True, + check=True, + ) + assert json.loads(read.stdout) == result + process.terminate() + _ = process.wait(timeout=5) + stale_id: str = str(uuid4()) + enqueue[enqueue.index(request_id)] = stale_id + _ = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + with location_process(arguments, log, publications) as process: + _ = wait_for_generation(api, "preview #1", 2, process, log) + result = wait_for_location_work(api, stale_id, process, log) + assert result["generation"] == 1 + assert result["result_json"]["outcome"] == {"kind": "superseded"} + changed_id: str = str(uuid4()) + enqueue[enqueue.index(stale_id)] = changed_id + blocked.touch() + _ = subprocess.run( + enqueue, env=owner_environment, capture_output=True, text=True, check=True + ) + _ = wait_for_interpreter(process, child_pid, log) + _ = (project / "models/stg_orders.sql").write_text( + TYPE_PROOF_FILES["models/stg_orders.sql"].replace( + "CAST(5 AS INTEGER)", "CAST(17 AS INTEGER)" + ) + ) + blocked.unlink() + result = wait_for_location_work(api, changed_id, process, log) + assert result["generation"] == 2 + assert result["result_json"]["outcome"] == {"kind": "superseded"} + _ = wait_for_generation(api, "preview #1", 3, process, log) + assert [ + json.loads(line)["generation"] for line in publications.read_text().splitlines() + ] == [1, 2, 3] + assert api.bearer not in log.read_text() + publications.read_text() + assert not (project / ".sqlbuild/control.sqlite3").exists() diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py new file mode 100644 index 0000000000..391f828b68 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -0,0 +1,839 @@ +"""Retained compiler baselines cross the native worker boundary before lifecycle writes.""" + +import json +import os +import signal +import sqlite3 +import time +from dataclasses import replace +from pathlib import Path +from typing import Any +from urllib.request import Request, urlopen +from uuid import uuid4 + +import duckdb +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + METADATA, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ClaimedWorker, + GroupExecutionTestCase, + SnapshotWorkerTestCase, + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + cancel_running_statement, + claimed_worker, + scheduled_worker, + snapshot_api, + wait_for_log, + wait_for_run, + worker_result, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + GroupExecutionTestCase( + "group execution preserves independent downstream work", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT amount FROM orders", + "partial", + [(11,)], + expected_child_statuses={"staging": ("partial",), "marts": ("partial",)}, + ), + GroupExecutionTestCase( + "successful jobs honor compiled boundaries", + "SELECT 17 AS amount", + "SELECT amount FROM totals", + "success", + [(17,)], + expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_totals=("success", None), + ), + GroupExecutionTestCase( + "a platform sensor starts group parents and children", + "SELECT 17 AS amount", + "SELECT amount FROM totals", + "success", + [(17,)], + expected_origin="sensor", + expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_totals=("success", None), + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import PlatformEventKind, job, job_group, platform_event, sensor\n" + "staging = job('staging', description='Stage orders.', select='stg_orders broken_orders')\n" + "marts = job('marts', description='Build independent marts.', select='orders totals')\n" + "annual_orders = job_group('annual_orders', description='Stage then build marts.', jobs=[marts, staging])\n" + "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "minimum_interval_seconds=60)\n" + ) + }, + ), + GroupExecutionTestCase( + "whole downstream job is blocked without building an empty selection", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "partial", + [(0,)], + on_failure="skip_downstream_jobs", + expected_child_statuses={"staging": ("partial",), "marts": ("failed",)}, + ), + GroupExecutionTestCase( + "job failure stops the group", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "failed", + [(0,)], + on_failure="stop_group", + expected_child_statuses={"staging": ("partial", "cancelled"), "marts": ("cancelled",)}, + expected_totals=("cancelled", None), + expected_marts_started=False, + ), + GroupExecutionTestCase( + "critical node stops the group under the default policy", + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)", + "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", + "failed", + [(0,)], + expected_child_statuses={"staging": ("partial", "cancelled"), "marts": ("cancelled",)}, + expected_totals=("cancelled", None), + expected_marts_started=False, + files={ + "models/broken_orders.sql": ( + "MODEL (description 'Critical orders.', materialized table, critical true);\n" + "SELECT CAST(value AS INTEGER) AS amount FROM (VALUES ('invalid')) AS source(value)\n" + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_failure_policy_hold( + test_case: GroupExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir: Path = tmp_path / "project" + project_dir.mkdir() + with duckdb.connect(str(project_dir / "warehouse.duckdb")) as warehouse: + warehouse.execute("CREATE TABLE broken_orders AS SELECT 99 AS amount") + test_case = replace( + test_case, + files={ + "models/orders.sql": ( + "MODEL (description 'Independent orders.', materialized table);\n" + 'SELECT amount FROM __ref("stg_orders")\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\nSELECT 11 AS amount\n" + ), + "models/broken_orders.sql": ( + "MODEL (description 'Source values.', materialized table);\n" + test_case.model_sql + ), + "models/totals.sql": ( + "MODEL (description 'Dependent totals.', materialized table);\n" + 'SELECT amount FROM __ref("broken_orders")\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + 'staging = job("staging", description="Stage orders.", ' + f'select="stg_orders broken_orders", on_failure="{test_case.on_failure}")\n' + 'marts = job("marts", description="Build independent marts.", select="orders totals")\n' + 'annual_orders = job_group("annual_orders", description="Stage then build marts.", ' + "jobs=[marts, staging], " + 'schedule=schedule(cron="0 0 1 1 *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + } + | test_case.files, + ) + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + parent: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert parent["status"] == test_case.expected_status, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: dict[str, Any] = {run["job_key"]: run for run in json.load(response)} + assert set(runs) == {"annual_orders", *test_case.expected_child_statuses} + assert all( + runs[name]["status"] in statuses + for name, statuses in test_case.expected_child_statuses.items() + ) + assert (runs["marts"]["started_at"] is not None) == test_case.expected_marts_started + assert runs["staging"]["ended_at"] <= ( + runs["marts"]["started_at"] or runs["staging"]["ended_at"] + ) + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT origin FROM sqb_runs WHERE run_id=?", (parent["run_id"],) + ).fetchone() == (test_case.expected_origin,) + assert ( + control.execute( + "SELECT parent_run_id, root_run_id FROM sqb_runs WHERE kind='job' ORDER BY job_key" + ).fetchall() + == [(runs["annual_orders"]["run_id"],) * 2] * 2 + ) + assert ( + control.execute( + "SELECT status, skipped_because_of FROM sqb_run_nodes WHERE run_id=? AND node_key='model:totals'", + (runs["marts"]["run_id"],), + ).fetchone() + == test_case.expected_totals + ) + assert control.execute( + "SELECT status FROM sqb_run_nodes WHERE run_id=? AND node_key='model:broken_orders'", + (runs["staging"]["run_id"],), + ).fetchone() == ({"success": "success"}.get(test_case.expected_status, "failed"),) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + with duckdb.connect(str(project_dir / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="opposing parent and independent child limits", + model_sql="SELECT 11 AS amount", + warehouse_query="SELECT amount FROM orders UNION ALL SELECT amount FROM inventory ORDER BY amount", + expected_status="success", + expected_rows=[(11,), (23,)], + concurrency_limits={"orders_limit": 1, "inventory_limit": 1}, + files={ + "models/inventory.sql": ( + "MODEL (description 'Inventory.', materialized table);\nSELECT 23 AS amount\n" + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "orders = job('orders_job', description='Build orders.', select='orders', " + "concurrency_groups=('inventory_limit',))\n" + "inventory = job('inventory_job', description='Build inventory.', select='inventory', " + "concurrency_groups=('orders_limit',))\n" + "annual_orders = job_group('annual_orders', description='Orders group.', jobs=[orders], " + "concurrency_groups=('orders_limit',), " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_inventory = job_group('annual_inventory', description='Inventory group.', " + "jobs=[inventory], concurrency_groups=('inventory_limit',), " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + store_setup=( + "CREATE TABLE test_slot_claims (run_id TEXT, group_key TEXT, occupied INTEGER);\n" + "CREATE TRIGGER test_observe_slots AFTER INSERT ON sqb_concurrency_slots BEGIN " + "INSERT INTO test_slot_claims SELECT NEW.run_id, NEW.group_key, count(*) " + "FROM sqb_concurrency_slots WHERE group_key=NEW.group_key; END;\n" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_opposing_group_limits_when_scheduled_then_both_real_workers_finish_without_hoisting_limits( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + orders: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + inventory: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_inventory" + ) + assert orders["status"] == inventory["status"] == "success", worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT r.job_key, c.group_key, max(c.occupied) FROM test_slot_claims c " + "JOIN sqb_runs r ON r.run_id=c.run_id GROUP BY r.job_key, c.group_key ORDER BY r.job_key" + ).fetchall() == [ + ("annual_inventory", "inventory_limit", 1), + ("annual_orders", "orders_limit", 1), + ("inventory_job", "orders_limit", 1), + ("orders_job", "inventory_limit", 1), + ] + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE status='success'" + ).fetchone() == (4,) + assert control.execute("SELECT count(*) FROM sqb_concurrency_slots").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotWorkerTestCase( + description="unselected dependency drift", + model_file="models/stg_orders.sql", + original_sql="CAST(5 AS INTEGER)", + replacement_sql="CAST(17 AS INTEGER)", + expected_error="node content differs", + ), + SnapshotWorkerTestCase( + description="selected model drift", + model_file="models/orders.sql", + original_sql="SELECT order_id, amount", + replacement_sql="SELECT order_id, amount * CAST(2 AS BIGINT) AS amount", + expected_error="node content differs", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_claim_lifetime_are_native( + test_case: SnapshotWorkerTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "project" + write_files(project_dir=project_dir, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + with snapshot_api(tmp_path / "control") as api: + metadata: dict[str, Any] = METADATA | {"location_id": api.location_id} + baseline: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, json.dumps(metadata), project_dir + ) + publication: dict[str, Any] = json.loads(baseline.publish(api.url, api.bearer)) + claim: ClaimedWorker = claimed_worker(tmp_path / "control", api, publication) + identity: tuple[str, str] = (claim.run_id, claim.claim_id) + changed: dict[str, str] = TYPE_PROOF_FILES | { + test_case.model_file: TYPE_PROOF_FILES[test_case.model_file].replace( + test_case.original_sql, test_case.replacement_sql + ) + } + write_files(project_dir=project_dir, files=changed) + drifted: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + wrong: NativeOrchestrationSnapshot = build_orchestration_snapshot( + drifted, json.dumps(metadata), project_dir + ) + with pytest.raises(RuntimeError, match=test_case.expected_error): + _ = wrong.start_worker(api.url, claim.bearer, identity) + runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" + location_headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + assert json.load(response)[0]["status"] == "queued" + with pytest.raises(RuntimeError, match="does not match"): + _ = baseline.start_worker(api.url, claim.bearer, (claim.run_id, str(uuid4()))) + with baseline.start_worker(api.url, claim.bearer, identity) as worker: + execution: dict[str, Any] = json.loads(worker.execution_json()) + assert execution["claim"]["request"]["selection"] == ["model:orders"] + assert execution["claim"]["request"]["settings"] == {"full_refresh": True} + assert frozenset(execution["node_hashes"]) == { + "model:orders", + "model:stg_orders", + } + assert not worker.cancel_requested() + event_id: str = str(uuid4()) + change: str = json.dumps({"kind": "node_started", "node_key": "model:orders"}) + assert worker.report(event_id, change) + assert not worker.report(event_id, change) + assert claim.bearer not in repr(worker) + worker.close() + assert worker.cancel_requested() + with pytest.raises(RuntimeError, match="closed"): + _ = worker.execution_json() + with pytest.raises(RuntimeError, match="closed"): + _ = worker.report(str(uuid4()), change) + with pytest.raises(RuntimeError, match="closed"), worker: + pytest.fail("closed worker cannot be entered") + # Closing an interpreter-side owner is not proof of physical warehouse termination. + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + assert json.load(response)[0]["status"] == "running" + with baseline.start_worker(api.url, claim.bearer, identity) as resumed: + assert resumed.report( + str(uuid4()), + json.dumps( + { + "kind": "node_finished", + "node_key": "model:orders", + "outcome": "success", + "rows_affected": 7, + "error_code": None, + "skipped_because_of": None, + } + ), + ) + assert resumed.report( + str(uuid4()), + json.dumps( + { + "kind": "finished", + "outcome": "success", + "reason": "worker execution completed", + "exit_code": 0, + } + ), + ) + with urlopen(Request(runs_url, headers=location_headers), timeout=5) as response: + result: dict[str, Any] = json.load(response)[0] + assert result["status"] == "success" + assert result["node_counts_json"] == {"success": 1} + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "successful model writes actual rows", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('23')) t(value)", + "SELECT amount FROM orders", + "success", + [(23,)], + ), + WorkerExecutionTestCase( + "a platform sensor catches up retained events through a real worker", + "SELECT 47 AS amount", + "SELECT amount FROM orders", + "success", + [(47,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor\n" + "annual_orders = job('annual_orders', select='orders', description='Build orders.')\n" + "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "minimum_interval_seconds=60)\n" + ) + }, + ), + WorkerExecutionTestCase( + "authored variables and reload use ordinary build argument conversion", + "SELECT @@batch AS amount", + "SELECT amount FROM orders", + "success", + [(23,)], + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n[vars]\nbatch = "3"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + "annual_orders = job('annual_orders', select='orders', description='Build orders.', " + "settings={'vars': {'batch': 23}, 'reload': True, 'run_tests': False}, " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', " + "starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ), + WorkerExecutionTestCase( + "warehouse error retains failed node evidence", + "SELECT CAST(value AS BIGINT) AS amount FROM (VALUES ('invalid')) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "failed", + [(0,)], + ), + WorkerExecutionTestCase( + "scalar function dependency is claimed and executed", + 'SELECT __udf("order_total")(23) AS amount', + "SELECT amount FROM orders", + "success", + [(46,)], + expected_node_count=2, + files={ + "functions/sql/order_total.sql": ( + "FUNCTION (description 'Order total.', " + "arguments (amount BIGINT), returns BIGINT);\nSELECT amount * 2\n" + ) + }, + ), + WorkerExecutionTestCase( + "table function dependency is claimed and executed", + 'SELECT amount FROM __table_fn("order_amounts")()', + "SELECT amount FROM orders", + "success", + [(31,)], + expected_node_count=2, + files={ + "functions/sql/order_amounts.sql": ( + "FUNCTION (description 'Order amounts.', returns table (amount BIGINT));\n" + "SELECT CAST(31 AS BIGINT) AS amount\n" + ) + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_real_warehouse_results( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "group completion starts a real sensor worker", + "SELECT 59 AS amount", + "SELECT amount FROM orders", + "success", + [(59,)], + expected_origin="sensor", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import RunEventKind, job, job_group, run_event, schedule, sensor\n" + "source = job('source_orders', select='orders', description='Source orders.')\n" + "source_group = job_group('source_group', description='Source group.', jobs=[source], " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_orders = job('annual_orders', select='orders', description='Refresh orders.', settings={'reload': True})\n" + "on_group = sensor('on_group', description='Refresh after group completion.', " + "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS], executed_successfully=('orders',)), " + "run=annual_orders, minimum_interval_seconds=1)\n" + ) + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destination_worker_executes( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + result: dict[str, Any] = worker_result(worker, test_case) + assert result["run_id"] == destination["run_id"] + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT run_key,run_id FROM sqb_run_requests").fetchall() == [ + ( + control.execute( + "SELECT run_id FROM sqb_runs WHERE job_key='source_group'" + ).fetchone()[0], + destination["run_id"], + ) + ] + assert control.execute( + "SELECT job_key,status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "success"), + ("source_group", "success"), + ("source_orders", "success"), + ] + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "group child node sensor", + "SELECT 67 AS amount", + "SELECT amount FROM orders", + "success", + [(67,)], + expected_origin="sensor", + files={ + "models/stg_orders.sql": "MODEL (description 'Staging orders.');\nSELECT 71 AS amount\n", + "orchestration/orders.py": ( + "from sqlbuild.orchestration import NodeEventKind, job, job_group, node_event, schedule, sensor\n" + "source = job('source_orders', select='orders', description='Source orders.')\n" + "source_group = job_group('source_group', description='Source group.', jobs=[source], " + "schedule=schedule(cron='0 0 1 1 *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "annual_orders = job('annual_orders', select='stg_orders', description='Refresh staging.', settings={'reload': True})\n" + "on_node = sensor('on_node', description='React to actual orders rebuilds.', " + "when=node_event(select='orders', kinds=[NodeEventKind.SUCCESS]), " + "run=annual_orders, minimum_interval_seconds=1)\n" + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_real_group_child_rebuild_when_a_node_sensor_matches_then_only_the_child_finish_queues_a_worker( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + destination: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert destination["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT job_key,status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "success"), + ("source_group", "success"), + ("source_orders", "success"), + ] + assert control.execute( + "SELECT q.run_key,q.run_id FROM sqb_run_requests q" + ).fetchall() == [ + ( + control.execute( + "SELECT e.event_id FROM recall_events e JOIN sqb_runs r ON r.run_id=e.run_id " + "WHERE r.job_key='source_orders' AND e.event_type='run_node_finished'" + ).fetchone()[0], + destination["run_id"], + ) + ] + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "cancellation stops an active warehouse statement", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'", + "cancelled", + [(0,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_stops_before_claim_release( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + cancel_running_statement(worker) + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "parent cancellation interrupts its active child and cancels its waiting child", + "SELECT sum(sin(value)) AS amount FROM range(10000000000) t(value)", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('orders','totals')", + "cancelled", + [(0,)], + files={ + "models/totals.sql": ( + "MODEL (description 'Downstream totals.', materialized table);\n" + 'SELECT amount FROM __ref("orders")\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, job_group, schedule\n" + "orders = job('orders', description='Long query.', select='orders')\n" + "totals = job('totals', description='Waiting query.', select='totals')\n" + "annual_orders = job_group('annual_orders', description='Cancellable group.', " + "jobs=[totals, orders], schedule=schedule(cron='0 0 1 1 *', timezone='UTC', " + "starts_at='2000-01-01T00:00:00Z'))\n" + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_executing_group_when_parent_cancelled_then_all_children_stop_before_parent_completion( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + parent: dict[str, Any] = wait_for_run( + worker.api, + worker.process, + worker.log, + pending=frozenset({"queued"}), + job_key="annual_orders", + ) + assert parent["status"] == "running", worker.log.read_text() + cancel_running_statement(worker, job_key="orders", run_id=parent["run_id"]) + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute( + "SELECT job_key, status FROM sqb_runs ORDER BY job_key" + ).fetchall() == [ + ("annual_orders", "cancelled"), + ("orders", "cancelled"), + ("totals", "cancelled"), + ] + assert control.execute( + "SELECT count(*) FROM sqb_runs WHERE parent_run_id=? AND ended_at>?", + (parent["run_id"], completed["ended_at"]), + ).fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "credential failure after a committed claim", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_run_token BEFORE INSERT ON sqb_access_tokens " + "WHEN NEW.run_id IS NOT NULL " + "BEGIN SELECT RAISE(ABORT, 'synthetic run credential failure'); END;" + ), + ), + WorkerExecutionTestCase( + "launch publication failure after a committed claim", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_launch BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status = 'launched' " + "BEGIN SELECT RAISE(ABORT, 'synthetic launch publication failure'); END;" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_committed_claim_when_launch_preparation_fails_then_unstarted_work_is_terminal_and_unreserved( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + assert worker.process.wait(timeout=15) != 0, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status, worker.log.read_text() + assert runs[0]["node_counts_json"] == {"skipped": 1} + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_concurrency_slots").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + assert not (worker.project / "warehouse.duckdb").exists() + assert "Worker launched for run" not in worker.log.read_text() + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "failed terminal publication keeps the owner alive until recovery", + "SELECT 23 AS amount", + "", + "failed", + [], + store_setup=( + "CREATE TRIGGER reject_run_token BEFORE INSERT ON sqb_access_tokens " + "WHEN NEW.run_id IS NOT NULL " + "BEGIN SELECT RAISE(ABORT, 'synthetic run credential failure'); END;" + "CREATE TRIGGER reject_terminal BEFORE UPDATE OF status ON sqb_runs " + "WHEN NEW.status = 'failed' " + "BEGIN SELECT RAISE(ABORT, 'synthetic terminal publication failure'); END;" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_stopped_worker_when_terminal_publication_fails_then_shutdown_retains_and_retries_its_claim( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + wait_for_log(worker.process, worker.log, "Retrying worker shutdown publication") + assert worker.process.poll() is None, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (1,) + _ = control.execute("DROP TRIGGER reject_terminal") + assert worker.process.wait(timeout=10) != 0, worker.log.read_text() + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + assert control.execute( + "SELECT count(*) FROM recall_events WHERE event_type='run_terminated'" + ).fetchone() == (1,) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX TERM/KILL escalation") +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "uncooperative child is reaped after a lease-preserving hard kill", + "SELECT 23 AS amount", + "", + "failed", + [], + launcher="uncooperative", + ) + ], + ids=lambda case: case.description, +) +def test_given_an_uncooperative_worker_when_scheduler_stops_then_the_fence_and_claims_survive_until_reaping( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + pidfile: Path = worker.project / "worker.pid" + deadline: float = time.monotonic() + 15 + while not pidfile.exists(): + assert worker.process.poll() is None, worker.log.read_text() + assert time.monotonic() < deadline, worker.log.read_text() + time.sleep(0.02) + pid: int = int(pidfile.read_text()) + worker.process.send_signal(signal.SIGTERM) + time.sleep(32) + assert worker.process.poll() is None, worker.log.read_text() + os.kill(pid, 0) + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (1,) + assert control.execute( + "SELECT julianday(expires_at) > julianday('now') FROM sqb_scheduler_lease" + ).fetchone() == (1,) + assert worker.process.wait(timeout=40) == 0, worker.log.read_text() + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + request: Request = Request( + f"{worker.api.url}/api/locations/{worker.api.location_id}/targets/development/runs", + headers={"Authorization": f"Bearer {worker.api.bearer}"}, + ) + with urlopen(request, timeout=5) as response: + runs: list[dict[str, Any]] = json.load(response) + assert len(runs) == 1 + assert runs[0]["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) + assert control.execute("SELECT count(*) FROM sqb_run_liveness").fetchone() == (0,) + assert control.execute( + "SELECT json_extract(payload, '$.change.change.exit_code') FROM recall_events " + "WHERE event_type='run_terminated'" + ).fetchone() == (137,) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py new file mode 100644 index 0000000000..6f6cc7a182 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -0,0 +1,1069 @@ +"""Real compilation produces immutable, relocatable control-plane catalog data.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + NativeOrchestrationSnapshot, + PythonNodeDiscoveryError, + build_compiled_project, + build_orchestration_snapshot, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.orchestration import ( + Job, + RunEventKind, + compile_snapshot, + job, + job_group, + run_event, + schedule, + sensor, +) +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + SnapshotCatalogTestCase, + SnapshotDeclarationRejectionTestCase, + SnapshotDiscoveryRejectionTestCase, + SnapshotDiscoveryTestCase, + SnapshotEnvironmentTestCase, + SnapshotRejectionTestCase, + SnapshotScheduleTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + FUNCTION_PROJECT_FILES, + METADATA, + PARITY_FILES, + TYPE_PROOF_FILES, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotCatalogTestCase( + "model and seed catalog", + TYPE_PROOF_FILES + | { + "seeds/payment_methods.csv": PARITY_FILES["seeds/payment_methods.csv"], + "seeds/payment_methods.yml": PARITY_FILES["seeds/payment_methods.yml"], + "models/payments.sql": ( + "MODEL (description 'Payment methods.');\n" + 'SELECT method FROM __seed("payment_methods")\n' + ), + }, + frozenset({"model", "seed"}), + ), + SnapshotCatalogTestCase( + "attached audits", + TYPE_PROOF_FILES + | { + "models/orders.sql": ( + "MODEL (description 'Orders.', columns (order_id (audits [not_null])));\n" + 'SELECT order_id, amount FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "audit"}), + ), + SnapshotCatalogTestCase( + "SQL and Python functions", + { + path: value + for path, value in FUNCTION_PROJECT_FILES.items() + if path != "enums/status.sql" + } + | {"functions/sql/_enums/status.sql": FUNCTION_PROJECT_FILES["enums/status.sql"]}, + frozenset({"model", "udf", "table_fn"}), + ), + SnapshotCatalogTestCase( + "Python hooks", + TYPE_PROOF_FILES + | { + "hooks/python/notifications.py": ( + "from sqlbuild.hooks import hook\n\n" + "@hook\ndef notify(ctx, channel):\n" + ' """Notify support."""\n return None\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', pre_hooks [python('notify', channel: 'support')]);\n" + 'SELECT order_id, amount FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "hook"}), + ), + SnapshotCatalogTestCase( + "SQL hooks", + TYPE_PROOF_FILES + | { + "hooks/sql/record_load.sql": "HOOK (description 'Record load.');\nSELECT 3 AS load_count\n", + "models/orders.sql": ( + "MODEL (description 'Orders.', pre_hooks [sql('record_load')]);\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + }, + frozenset({"model", "sql_hook"}), + ), + SnapshotCatalogTestCase( + "source catalog", + TYPE_PROOF_FILES + | { + "sources/orders.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: raw\n table: orders\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + }, + frozenset({"model", "source"}), + ), + SnapshotCatalogTestCase( + "DuckShop polling callback with nested code is relocatable", + TYPE_PROOF_FILES + | { + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" + ).read_text(), + }, + frozenset({"model"}), + ), + SnapshotCatalogTestCase( + "DuckShop inherited seed and function namespaces", + TYPE_PROOF_FILES + | { + path: (Path("benchmarks/duckshop/project") / path).read_text() + for path in ( + "seeds/edge_cases/edge_quantity_steps.csv", + "seeds/edge_cases/edge_quantity_steps.yml", + "functions/sql/edge_cases/edge_next_quantity.sql", + ) + }, + frozenset({"model", "seed", "udf"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_real_projects_when_building_snapshots_then_catalogs_are_stable_and_relocatable( + test_case: SnapshotCatalogTestCase, + tmp_path: Path, +) -> None: + publications: list[dict[str, Any]] = [] + metadata: str = json.dumps(METADATA) + for directory in (tmp_path / "original", tmp_path / "relocated"): + write_files(project_dir=directory, files=test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=directory), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert not any(item.severity == "error" for item in project.diagnostics) + snapshot: NativeOrchestrationSnapshot = build_orchestration_snapshot( + project, metadata, directory + ) + publication: dict[str, Any] = json.loads(snapshot.to_json()) + publications.append(publication) + assert publication["sqb_version"] != "not-the-installed-version" + assert publication["environment_hash"] != "test-environment" + assert test_case.expected_kinds <= {node["kind"] for node in publication["nodes"].values()} + assert str(directory) not in snapshot.to_json() + for model in project.models: + node: dict[str, Any] = publication["nodes"][f"model:{model.name}"] + assert node["compiled_sql"] == model.query_sql + assert ( + node["definition"]["destination"]["qualified_name"] + == model.destination.qualified_name + ) + assert set(node["definition"]["deps"]) == { + f"{dependency.resource_type}:{dependency.name}" for dependency in model.deps + } + for function in project.functions: + assert ( + publication["nodes"][f"{function.key.resource_type}:{function.name}"]["definition"][ + "destination" + ]["qualified_name"] + == function.destination.qualified_name + ) + for seed in project.seeds: + assert ( + publication["nodes"][f"seed:{seed.name}"]["definition"]["destination"][ + "qualified_name" + ] + == seed.destination.qualified_name + ) + assert publications[0] == publications[1] + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotRejectionTestCase( + description="a valid compile cannot publish caller-forged nodes", + files=TYPE_PROOF_FILES, + nodes={ + "model:forged": { + "kind": "model", + "definition": {}, + "raw_sql": None, + "compiled_sql": "select 2", + "python_source": None, + } + }, + expected_compiler_errors=False, + expected_message="nodes must come from the compiled project", + ), + SnapshotRejectionTestCase( + description="compiler errors cannot replace the last good publication", + files=TYPE_PROOF_FILES | {"models/orders.sql": "MODEL ();\nSELECT 1 AS order_id\n"}, + nodes={}, + expected_compiler_errors=True, + expected_message="cannot publish a project with compiler errors", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snapshot_is_emitted( + test_case: SnapshotRejectionTestCase, + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert ( + any(item.severity == "error" for item in project.diagnostics) + == test_case.expected_compiler_errors + ) + with pytest.raises(ValueError, match=test_case.expected_message): + build_orchestration_snapshot( + project, + json.dumps(METADATA | {"nodes": test_case.nodes}), + tmp_path, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotEnvironmentTestCase( + description="installed dependencies and two lock revisions identify different environments", + first_lockfile="version = 1\n# dependency revision one\n", + second_lockfile="version = 1\n# dependency revision two\n", + expected_environment_count=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_dependency_lock_when_it_changes_then_environment_identity_changes_not_node_content( + test_case: SnapshotEnvironmentTestCase, + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + installed: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + (tmp_path / "uv.lock").write_text(test_case.first_lockfile) + locked: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + (tmp_path / "uv.lock").write_text(test_case.second_lockfile) + changed: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + assert ( + len( + {installed["environment_hash"], locked["environment_hash"], changed["environment_hash"]} + ) + == test_case.expected_environment_count + ) + assert installed["nodes"] == locked["nodes"] == changed["nodes"] + assert changed == json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotScheduleTestCase( + description="daylight saving zone", + schedule=schedule(cron="15 2 * * *", timezone="Europe/London"), + expected_selection=("model:orders",), + ), + SnapshotScheduleTestCase( + description="fixed offset zone", + schedule=schedule(cron="15 2 * * *", timezone="UTC"), + expected_selection=("model:orders",), + ), + SnapshotScheduleTestCase( + description="paused bounded catch-up with a zoned start", + schedule=schedule( + cron="15 2 * * *", + timezone="Europe/London", + starts_at="2026-07-03T01:17:23.456789+05:30", + default_status="paused", + catch_up="all", + max_catch_up=3, + ), + expected_selection=("model:orders",), + expected_controls={ + "starts_at": "2026-07-02T19:47:23.456789Z", + "default_status": "paused", + "catch_up": {"all": {"max_runs": 3}}, + }, + ), + SnapshotScheduleTestCase( + description="largest supported catch-up batch", + schedule=schedule( + cron="15 2 * * *", timezone="UTC", catch_up="all", max_catch_up=10_000 + ), + expected_selection=("model:orders",), + expected_controls={ + "starts_at": None, + "default_status": "running", + "catch_up": {"all": {"max_runs": 10_000}}, + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_selection_and_cron( + tmp_path: Path, test_case: SnapshotScheduleTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + nightly: Job = job( + "nightly_orders", + description="Refresh order reporting.", + select="+orders", + exclude="stg_orders", + overlap="queue", + settings={"run_tests": False, "vars": {"batch": 3}, "warehouse": "reporting"}, + concurrency=4, + concurrency_groups=("order_reporting",), + schedule=test_case.schedule, + targets=("development",), + ) + excluded: Job = replace(nightly, name="other_target", targets=("production",)) + first: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[nightly, excluded], + ).to_json() + ) + assert first["documents"]["jobs"] == { + "nightly_orders": { + "description": "Refresh order reporting.", + "selection": list(test_case.expected_selection), + "settings": { + "concurrency": 4, + "run_tests": False, + "vars": {"batch": 3}, + "warehouse": "reporting", + }, + "overlap": "queue", + "on_failure": "skip_dependents", + "concurrency_groups": ["order_reporting"], + } + } + trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders"] + assert trigger["job_key"] == "nightly_orders" + assert trigger["schedule"] == { + "kind": "cron", + "expression": "15 2 * * *", + "timezone": test_case.schedule.timezone, + } + assert {key: trigger[key] for key in test_case.expected_controls} == test_case.expected_controls + edited: Job = replace(nightly, schedule=schedule(cron="30 3 * * *", timezone="UTC")) + second: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[edited], + ).to_json() + ) + assert second["documents"]["triggers"].keys() == first["documents"]["triggers"].keys() + assert second["documents"]["triggers"]["nightly_orders"]["schedule"]["expression"] == ( + "30 3 * * *" + ) + assert second["nodes"] == first["nodes"] + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDeclarationRejectionTestCase( + description="blank description", + job_description=" ", + expected_message="needs a description", + ), + SnapshotDeclarationRejectionTestCase( + description="blank selection", select="", expected_message="needs a selection" + ), + SnapshotDeclarationRejectionTestCase( + description="missing model", select="missing", expected_message="missing" + ), + SnapshotDeclarationRejectionTestCase( + description="invalid cron", cron="not a cron", expected_message="cron" + ), + SnapshotDeclarationRejectionTestCase( + description="invalid timezone", timezone="Unknown/Zone", expected_message="time zone" + ), + SnapshotDeclarationRejectionTestCase( + description="zero catch-up bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=0), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="negative catch-up bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=-1), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="boolean is not a catch-up bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=True), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="unbounded all catch-up", + schedule=schedule(cron="0 2 * * *", timezone="UTC", catch_up="all"), + expected_message="catch_up", + ), + SnapshotDeclarationRejectionTestCase( + description="catch-up exceeds transaction bound", + schedule=schedule( + cron="0 2 * * *", timezone="UTC", catch_up="all", max_catch_up=10_001 + ), + expected_message="catch-up exceeds 10000", + ), + SnapshotDeclarationRejectionTestCase( + description="latest does not silently ignore a bound", + schedule=schedule(cron="0 2 * * *", timezone="UTC", max_catch_up=3), + expected_message="latest.*without a bound", + ), + SnapshotDeclarationRejectionTestCase( + description="start needs an explicit offset", + schedule=schedule(cron="0 2 * * *", timezone="UTC", starts_at="2026-07-03T01:17:23"), + expected_message="starts_at", + ), + SnapshotDeclarationRejectionTestCase( + description="duplicate names", copies=2, expected_message="duplicate job orders" + ), + SnapshotDeclarationRejectionTestCase( + description="unknown setting", + settings={"concurreny": 4}, + expected_message="unknown build setting concurreny", + ), + SnapshotDeclarationRejectionTestCase( + description="selection cannot escape its pinned run", + settings={"select": "customers"}, + expected_message="owned by the run request", + ), + SnapshotDeclarationRejectionTestCase( + description="zero concurrency", + concurrency=0, + expected_message="must be positive", + ), + SnapshotDeclarationRejectionTestCase( + description="boolean is not concurrency", + concurrency=True, + expected_message="concurrency must be an integer", + ), + SnapshotDeclarationRejectionTestCase( + description="integer is not a boolean flag", + settings={"full_refresh": 1}, + expected_message="must be boolean", + ), + SnapshotDeclarationRejectionTestCase( + description="integer strings are not typed settings", + settings={"concurrency": "3"}, + expected_message="must be an integer", + ), + SnapshotDeclarationRejectionTestCase( + description="variables need an object", + settings={"vars": ["orders"]}, + expected_message="must be a JSON object", + ), + SnapshotDeclarationRejectionTestCase( + description="mutually exclusive load settings", + settings={"load_sources": False, "reload": True}, + expected_message="mutually exclusive", + ), + SnapshotDeclarationRejectionTestCase( + description="conflicting cursor families", + settings={"start_cursor": "2", "end_cursor_int": "4"}, + expected_message="cannot be combined", + ), + SnapshotDeclarationRejectionTestCase( + description="duplicate concurrency declarations", + concurrency=3, + settings={"concurrency": 4}, + expected_message="declared twice", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_declarations_when_compiling_then_native_validation_rejects_them( + tmp_path: Path, test_case: SnapshotDeclarationRejectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + declarations: list[Job] = [ + job( + "orders", + description=test_case.job_description, + select=test_case.select, + settings=test_case.settings, + concurrency=test_case.concurrency, + schedule=test_case.schedule + or schedule(cron=test_case.cron, timezone=test_case.timezone), + ) + ] * test_case.copies + with pytest.raises(ValueError, match=test_case.expected_message): + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=declarations, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDiscoveryTestCase( + description="relative imports aliases and factory containers", + files={ + "orchestration/shared.py": ( + "from sqlbuild.orchestration import job\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + ), + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "from .shared import staging\n" + "alias = staging\n" + "def factory():\n" + " return [job('daily', description='Daily orders.', select='orders')]\n" + "jobs = [factory(), staging]\n" + "jobs.append(jobs)\n" + ), + }, + expected_jobs=("daily", "staging"), + ), + SnapshotDiscoveryTestCase( + description="group-only references retain lineage and explicit waits", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "def factory():\n" + " staging = job('staging', description='Stage orders.', select='stg_orders')\n" + " marts = job('marts', description='Build orders.', select='orders')\n" + " inventory = job('inventory', description='Repeat staging.', select='stg_orders')\n" + " return job_group('daily', description='Ordered jobs.', " + "jobs=[marts, inventory, staging], after={inventory: [staging]})\n" + "daily = factory()\n" + ), + }, + expected_jobs=("inventory", "marts", "staging"), + expected_groups={ + "daily": { + "inventory": ["staging"], + "marts": ["inventory", "staging"], + "staging": [], + } + }, + ), + SnapshotDiscoveryTestCase( + description="explicit empty declarations do not execute project files", + files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, + expected_jobs=(), + declarations=(), + ), + SnapshotDiscoveryTestCase( + description="DuckShop edge-case declaration containers", + files={ + "orchestration/edge_cases/definitions.py": Path( + "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" + ).read_text(), + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + }, + expected_jobs=( + "edge_snapshot_inventory", + "edge_snapshot_orders", + "edge_snapshot_products", + ), + expected_groups={ + "edge_snapshot_group": {"edge_snapshot_orders": []}, + "edge_inventory_group": {"edge_snapshot_inventory": []}, + "edge_products_group": {"edge_snapshot_products": []}, + }, + expected_sensors={ + "edge_platform_orders": { + "description": "Catch up retained platform observations with a stopped group sensor.", + "job_key": "edge_snapshot_group", + "when": { + "kind": "platform_event", + "kinds": ["tick_overdue", "location_broken", "launch_failed"], + }, + "minimum_interval_seconds": 7, + "default_status": "stopped", + }, + "edge_job_outcomes": { + "description": "Keep warning and partial outcomes distinct from success.", + "job_key": "edge_snapshot_inventory", + "when": { + "kind": "run_event", + "job_key": "edge_snapshot_orders", + "kinds": ["warn", "partial"], + }, + "minimum_interval_seconds": 30, + "default_status": "stopped", + }, + "edge_group_outcomes": { + "description": "Observe group completion including cancellation and presumed failure.", + "job_key": "edge_snapshot_inventory", + "when": { + "kind": "run_event", + "job_key": "edge_snapshot_group", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], + }, + "minimum_interval_seconds": 30, + "default_status": "stopped", + }, + }, + expected_schedules=("edge_snapshot_orders",), + ), + SnapshotDiscoveryTestCase( + description="sensor-only roots collect source groups and destination jobs natively", + files={}, + declarations=( + sensor( + "on_orders", + description="Build inventory after group completion.", + when=run_event( + job=job_group( + "source_group", + description="Source orders.", + jobs=[job("source_orders", description="Orders.", select="orders")], + ), + kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders", "orders*"), + ), + run=job("inventory", description="Inventory.", select="stg_orders"), + ), + ), + expected_jobs=("inventory", "source_orders"), + expected_groups={"source_group": {"source_orders": []}}, + expected_sensors={ + "on_orders": { + "description": "Build inventory after group completion.", + "job_key": "inventory", + "when": { + "kind": "run_event", + "job_key": "source_group", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="any-run node selectors freeze canonical keys without a source job reference", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ).read_text() + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory for every genuine orders rebuild.", + "job_key": "inventory", + "when": { + "kind": "node_event", + "selection": ["model:orders"], + "kinds": ["success"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="source-job execution selectors compile to frozen canonical keys", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ).read_text() + }, + expected_jobs=("inventory", "source_orders"), + expected_sensors={ + "on_orders": { + "description": "Build inventory only after orders actually succeeds.", + "job_key": "inventory", + "when": { + "kind": "run_event", + "job_key": "source_orders", + "kinds": ["completed"], + "executed_successfully": ["model:orders"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + } + }, + ), + SnapshotDiscoveryTestCase( + description="sensor-only roots preserve referenced jobs and filter targets", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ).read_text() + }, + expected_jobs=("health_orders",), + expected_sensors={ + "overdue_orders": { + "description": "Catch up overdue order reporting.", + "job_key": "health_orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 7, + "default_status": "stopped", + } + }, + ), + SnapshotDiscoveryTestCase( + description="a project without declarations has no job document", + files={}, + expected_jobs=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_declaration_modules_when_compiling_then_native_discovery_collects_jobs( + tmp_path: Path, test_case: SnapshotDiscoveryTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + publication: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=test_case.declarations, + ).to_json() + ) + assert tuple(publication["documents"].get("jobs", {})) == test_case.expected_jobs + assert { + name: definition["jobs"] + for name, definition in publication["documents"].get("groups", {}).items() + } == test_case.expected_groups + assert { + name: publication["documents"]["triggers"][name] for name in test_case.expected_sensors + } == test_case.expected_sensors + assert tuple(publication["documents"].get("triggers", {})) == tuple( + sorted((*test_case.expected_schedules, *test_case.expected_sensors)) + ) + for definition in publication["documents"].get("jobs", {}).values(): + assert set(definition["selection"]) <= publication["nodes"].keys() + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotDiscoveryRejectionTestCase( + description="group members are jobs, not nested groups", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/nested_groups.py" + ).read_text(), + }, + expected_error=TypeError, + expected_message="group members must be Job objects", + ), + SnapshotDiscoveryRejectionTestCase( + description="a polling sensor needs an actual callback", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, sensor\n" + "orders = job('orders', description='Build orders.', select='orders')\n" + "poll = sensor('poll', description='Poll orders.', run=orders)\n" + ), + }, + expected_error=ValueError, + expected_message="either a built-in condition or a decorated polling callback", + ), + SnapshotDiscoveryRejectionTestCase( + description="callbacks with event conditions remain explicitly unsupported", + files={ + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, sensor, platform_event, PlatformEventKind\n" + "orders = job('orders', description='Build orders.', select='orders')\n" + "@sensor('poll', description='Poll orders.', run=orders, " + "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]))\n" + "def poll(ctx):\n return []\n" + ), + }, + expected_error=ValueError, + expected_message="callbacks with when are not supported yet", + ), + SnapshotDiscoveryRejectionTestCase( + description="explicit group cycles cannot publish", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "marts = job('marts', description='Build orders.', select='orders')\n" + "daily = job_group('daily', description='Cyclic jobs.', " + "jobs=[marts, staging], after={staging: [marts]})\n" + ), + }, + expected_error=ValueError, + expected_message="group dependencies contain a cycle", + ), + SnapshotDiscoveryRejectionTestCase( + description="group waits cannot refer to nonmember jobs", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job, job_group\n" + "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "marts = job('marts', description='Build orders.', select='orders')\n" + "daily = job_group('daily', description='Unknown upstream.', " + "jobs=[marts], after={marts: [staging]})\n" + ), + }, + expected_error=ValueError, + expected_message="group needs known jobs and dependencies within its membership", + ), + SnapshotDiscoveryRejectionTestCase( + description="distinct objects cannot share a job name", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "jobs = [job('daily', description='Orders.', select='orders') for _ in range(2)]\n" + ), + }, + expected_error=ValueError, + expected_message="duplicate job daily", + ), + SnapshotDiscoveryRejectionTestCase( + description="broken imports fail on every retry", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/failed_import.py" + ).read_text() + }, + expected_error=PythonNodeDiscoveryError, + expected_message="Failed to import orchestration file orchestration/jobs.py", + ), + SnapshotDiscoveryRejectionTestCase( + description="DuckShop settings cannot replace the selection", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py" + ).read_text() + }, + expected_error=ValueError, + expected_message="owned by the run request", + ), + SnapshotDiscoveryRejectionTestCase( + description="DuckShop mutually exclusive settings fail on every retry", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py" + ).read_text() + }, + expected_error=ValueError, + expected_message="mutually exclusive", + ), + SnapshotDiscoveryRejectionTestCase( + description="a zero sensor interval cannot publish", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("minimum_interval_seconds=7", "minimum_interval_seconds=0") + }, + expected_error=ValueError, + expected_message="nonzero u32", + ), + SnapshotDiscoveryRejectionTestCase( + description="empty platform conditions cannot publish", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=[]") + }, + expected_error=ValueError, + expected_message="nonempty event kinds", + ), + SnapshotDiscoveryRejectionTestCase( + description="platform kinds are a closed vocabulary", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=['unknown_event']") + }, + expected_error=ValueError, + expected_message="unknown variant", + ), + SnapshotDiscoveryRejectionTestCase( + description="empty node selectors cannot broaden matching to all nodes", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace('select="orders orders*"', 'select=" "') + }, + expected_error=ValueError, + expected_message="node_event needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="node event kinds do not imply unapproved failure transitions", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + ) + .read_text() + .replace("kinds=[NodeEventKind.SUCCESS]", "kinds=['failed']") + }, + expected_error=ValueError, + expected_message="unknown variant", + ), + SnapshotDiscoveryRejectionTestCase( + description="explicit empty execution selectors cannot broaden matching", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', "()") + }, + expected_error=ValueError, + expected_message="executed_successfully needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="blank execution selectors cannot broaden matching", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '(" ",)') + }, + expected_error=ValueError, + expected_message="executed_successfully needs nonempty selectors", + ), + SnapshotDiscoveryRejectionTestCase( + description="unresolved execution selectors fail closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("unknown_orders",)') + }, + expected_error=ValueError, + expected_message="unknown selector name 'unknown_orders'", + ), + SnapshotDiscoveryRejectionTestCase( + description="unmatched execution patterns cannot become an omitted filter", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("unknown_*",)') + }, + expected_error=ValueError, + expected_message="unknown selector pattern", + ), + SnapshotDiscoveryRejectionTestCase( + description="invalid execution selector syntax fails closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("tag:",)') + }, + expected_error=ValueError, + expected_message="has empty value after ':'", + ), + SnapshotDiscoveryRejectionTestCase( + description="execution selectors outside the source job fail closed", + files={ + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + ) + .read_text() + .replace('("orders",)', '("stg_orders",)') + }, + expected_error=ValueError, + expected_message="executed_successfully must select nonempty source job/group nodes", + ), + SnapshotDiscoveryRejectionTestCase( + description="sensor code defaults use running or stopped", + files={ + "orchestration/health.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + ) + .read_text() + .replace('default_status="stopped"', 'default_status="paused"') + }, + expected_error=ValueError, + expected_message="unknown variant", + ), + ], + ids=lambda case: case.description, +) +def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapshot_is_returned( + tmp_path: Path, test_case: SnapshotDiscoveryRejectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + for _ in range(2): + with pytest.raises(test_case.expected_error, match=test_case.expected_message): + compile_snapshot( + project=project, metadata_json=json.dumps(METADATA), project_dir=tmp_path + ) diff --git a/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py b/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py index dd3e4ba20f..9422bcd69a 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py +++ b/tests/unit/scripts/fensu_policy/rules/test_native_boundary.py @@ -135,6 +135,37 @@ def test_given_test_module_when_checking_native_patches_then_matches_contract( ), expected_lines=(3, 4, 5, 5, 6, 6), ), + NativeLinesTestCase( + description="pyclass module metadata is not a Python crossing", + source='#[pyclass(frozen, module = "sqlbuild._native")]\nstruct Snapshot;\n', + expected_lines=(), + ), + NativeLinesTestCase( + description="multiline namespaced pyclass metadata preserves callback line numbers", + source=dedent( + """ + #[pyo3::pyclass( + frozen, + module = "sqlbuild._native", + )] + struct Snapshot; + fn read(py: Python<'_>) { + py.import("sqlbuild.compiler"); + } + """ + ), + expected_lines=(8, 8), + ), + NativeLinesTestCase( + description="raw pyclass module metadata does not hide an adjacent real crossing", + source='#[pyclass(module = r#"sqlbuild._native"#)] struct Snapshot; fn read() { imported(py, "sqlbuild.compiler", "compile"); }\n', + expected_lines=(1, 1), + ), + NativeLinesTestCase( + description="module assignments outside pyclass remain crossings", + source='fn read() { let module = "sqlbuild.compiler"; load(module); }\n', + expected_lines=(1,), + ), NativeLinesTestCase( description="comments, imports and test modules are skipped", source=dedent( diff --git a/uv.lock b/uv.lock index 7f698d1833..d0a6aa78ca 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.28.0" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/69/a132ffc407be97a0fdcfa052cb8b49408091db748e6afe0f7a6c77e6fa7d/fensu-0.28.0.tar.gz", hash = "sha256:11bcea3207f7fd92babc200f40d074e3da789ee11e7fc7024575a415de7c1176", size = 538990, upload-time = "2026-10-27T00:39:16.19Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/3d/d5b4fb0ad09426dae015af2817ba8416d2ea974a0c4a6ebea63112f90b1b/fensu-0.28.3.tar.gz", hash = "sha256:a7fc87cb60eaef964b4e980d19bd6de9c95cecc75c019d8b0058e02b6751f4b5", size = 539031, upload-time = "2026-11-01T09:41:23.715Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/e9/7434bd0565d54ff58796105728b3ccc8b838c7200997184978e6ebfdec4d/fensu-0.28.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:29df31aff900d20b0c845940af0cd91eead55e158a2511e682e3e887a22dada8", size = 4154207, upload-time = "2026-10-27T00:38:52.311Z" }, - { url = "https://files.pythonhosted.org/packages/7f/d8/e0875b269515d9a75643d5488bdc6f70a4b6c346995f12ce07304a3610f7/fensu-0.28.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:33a6d8b7d2f734c48f48087462c0e5a1ff94535c0e27e6a4820bc10303ff3653", size = 4004243, upload-time = "2026-10-27T00:38:54.994Z" }, - { url = "https://files.pythonhosted.org/packages/e8/05/b87c0ca5cf3f724517d6958f1ebd68948f9399306bd6d6814783635bad9a/fensu-0.28.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de07b3fee5d908d2a766ca136753c8040d6fe25a1346d9b3b5a98483fc23d874", size = 4230192, upload-time = "2026-10-27T00:38:57.321Z" }, - { url = "https://files.pythonhosted.org/packages/cb/d2/fdfc7fd322c3497c53c22754a33433e33678a6de84832fd41add09481a21/fensu-0.28.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f44293a2da6bf4558d00eb4f859b28f3fb69d396dad1c25ac466a92a7d1dd734", size = 4307267, upload-time = "2026-10-27T00:38:59.679Z" }, - { url = "https://files.pythonhosted.org/packages/56/c0/2ab2c2c1af4ddd1b45c750679bd9f0b777d7eca3c0e00a233c991aaa13e7/fensu-0.28.0-cp312-abi3-win_amd64.whl", hash = "sha256:ffa88f1d259a1830d513ae930ff856d5b5e4941ba433889dea04d3db5753f281", size = 3977828, upload-time = "2026-10-27T00:39:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a7/9493fbcf3158ae009651d19c7289855e8ef571dfaf5c7c77154a8a6a4a21/fensu-0.28.3-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e3e3896f173d2bb676fe04defdd4ebdf4db38aee3ee6e57d7a5a297df32d7c5c", size = 4154274, upload-time = "2026-11-01T09:41:03.614Z" }, + { url = "https://files.pythonhosted.org/packages/a2/71/20fdb7147e268c5dbd9eae0fc9dd99641e416f9e75e8fd067ac089f6db78/fensu-0.28.3-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d74db7466b2ddd533413d2daf4c704aa4acbcae61e2a0edba2564a4910666707", size = 4004309, upload-time = "2026-11-01T09:41:05.719Z" }, + { url = "https://files.pythonhosted.org/packages/d0/4c/40b01d8e46ff3464eb503529e00ad23eddd2ffbd605282618672d0aff45f/fensu-0.28.3-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da5843aa40c7d6398a95b340b3c60d4360b5feddac34960663ee84aec09df8ef", size = 4230261, upload-time = "2026-11-01T09:41:07.481Z" }, + { url = "https://files.pythonhosted.org/packages/e7/7a/243cccd83531c3c86c660b993e77a7014c65d1d3f20c07e45b067dba81cf/fensu-0.28.3-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:52e1623e614f101c5ada2eda59548ff441b7efe4228813e69bc0f0ef9b3fbca6", size = 4307334, upload-time = "2026-11-01T09:41:09.938Z" }, + { url = "https://files.pythonhosted.org/packages/88/a3/38dbb50d5079a1a5fcaf62bcdd582c12a53609c575a747cb2461e83ddfcf/fensu-0.28.3-cp312-abi3-win_amd64.whl", hash = "sha256:a39c407725ea4d92123414bdbe9f80d5db0b7a412af49082a87378203c9ecb04", size = 3977904, upload-time = "2026-11-01T09:41:11.672Z" }, ] [[package]] name = "fensu-cli" -version = "0.28.0" +version = "0.28.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/d9/f347f514fb224f6a1f4ca6e616ac49b573f9dceba4ff8d9efcd334f48bea/fensu_cli-0.28.0.tar.gz", hash = "sha256:1feb379ed92e193cbd4d28ab3d7729798b88120aaee2f59fd7f341519b6ee945", size = 765638, upload-time = "2026-10-27T00:39:17.752Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/17/3d0953c33387caa0a6becc9bb02210d4a020eaaf73ef07bdc0c8f8952562/fensu_cli-0.28.3.tar.gz", hash = "sha256:3ba38bedc88a9e6db8563f638aa6ba8649a3e5750de176e22a7b35561d14bd9e", size = 766571, upload-time = "2026-11-01T09:41:25.271Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/b3/57f1de89e1cb7534b0ef6415822c8982a247b9d2efe8d2fddd85e35389c8/fensu_cli-0.28.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:13ecc7a224a4b32eef19456f21659e682e636ef53f305919aa57cae27c8e3fec", size = 7978247, upload-time = "2026-10-27T00:39:04.644Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ff/2e887d727f82c61ea7cb6773429b61d4711b93f7f94814eb9a7e891384dc/fensu_cli-0.28.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:149e5e22c8895c0a3ed3956b30e693e78df7dfb67ce8e0c5682b84c77e811e7e", size = 7490546, upload-time = "2026-10-27T00:39:07.384Z" }, - { url = "https://files.pythonhosted.org/packages/76/4e/daa4c2d70e5b0398382f84c846aacc4a937c4d00b3938d00a67351b0f212/fensu_cli-0.28.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:99932d17956bb10da1ac677867567b504a961d0565fcda730c1e682434244113", size = 7570220, upload-time = "2026-10-27T00:39:09.852Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f9/c9e1095c507da3a53b3e7bf3f7c95f75824429ea45a72cb2c7af67a605f7/fensu_cli-0.28.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee51c5d6d0d5fa0b36c0f70d3d6fde429793b643f0075cdfdf3fc8b8d73e3ccb", size = 8086364, upload-time = "2026-10-27T00:39:12.052Z" }, - { url = "https://files.pythonhosted.org/packages/9c/89/ece32e452d2b4ade243dd66a2612619be7400f6acf59f854ce674bb78e30/fensu_cli-0.28.0-py3-none-win_amd64.whl", hash = "sha256:acca6ac39cbe0614d3a8e4ce7c2c12cc5fd3c800fc4c66b5484c54dc19ef4a66", size = 7700477, upload-time = "2026-10-27T00:39:14.15Z" }, + { url = "https://files.pythonhosted.org/packages/ff/bb/09753cc0584c21af1d76eca7c9a6ea5e2d876d71c079c5505dda93742f3d/fensu_cli-0.28.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5179b032074f4ff5097950c0c0b063fd872cffcd23a4bf8a89c71ab0d3d8074b", size = 7963143, upload-time = "2026-11-01T09:41:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/47/59/663fb61d56172bfb053375e8623a23db389b746052fb00dd087cf6c1a76b/fensu_cli-0.28.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:41077b9ec6e2eb46405af809d1010b26519c8928d4264137ca4c6d47ce81d2e1", size = 7479791, upload-time = "2026-11-01T09:41:16.253Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7c/1a03bd25bc2a231ca03478ba3dfd8f21c909eae564c3df751a8982222beb/fensu_cli-0.28.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ac929d598046846c18eae62c8b75a9a49e9b6bbcadb46bd413a5da96ef6453e2", size = 7854377, upload-time = "2026-11-01T09:41:18.109Z" }, + { url = "https://files.pythonhosted.org/packages/17/5e/8395e4bd4e9e4d282a0949e42e721101127fb30f45cb71d6e5af05329051/fensu_cli-0.28.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ce453889008381184a416acaf274ffa4a46c7cb96ad4397de264816bb10626a", size = 8316670, upload-time = "2026-11-01T09:41:20.045Z" }, + { url = "https://files.pythonhosted.org/packages/89/40/6e3831cd39b16f163916b36c03010754c1cfb58583fd737517bd544751d5/fensu_cli-0.28.3-py3-none-win_amd64.whl", hash = "sha256:66ec08187a6dcc43392244ba4b7349ee29e8da63fbe7110114468f07a000d577", size = 7710605, upload-time = "2026-11-01T09:41:22.073Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.28.0" }, + { name = "fensu", specifier = ">=0.28.3" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" },